From fdebebc514e9812f2442d5bcd3ec4a79f494d3a9 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 26 May 2026 11:48:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..4e98628 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.3 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..d4f8170 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1186340892367426, + "train_runtime": 1737.4974, + "train_samples": 43598, + "train_samples_per_second": 25.092, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..61da035 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f3e1c4ec998d3dc1ae1d857f48c595480ec9db2f31f3724eede48e8f378e419 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..a5557ac --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:591fe4cc7945376216c5872e3a4307910ea2d134d09150b3c2115e139ff3ce72 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..ce8d650 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe55cc9f74aec4538db6bf8b7b09b792a3f2321ac86583bf06373ff95339f46 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..bb793ed --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b0c4bc2aa2cffbb2718a614ac1e54bddae7f764483a271b82e29e506e979788 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..e5fd40d --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4c50ecd06dff2a3385cb4255c1b3f04f75a4d2b79dbb4ea55ea7100d72b4f2 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..e2b4e55 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75bb32b54d12b12aa8bac1a0356b6ca544ffc12471e6a2dee83b1535499a8b35 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..a60d9a9 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e673c19ccb41dd8044b1c4bb4d05634525bb052263eb14ed8815f645f86a73f +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..d4f8170 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1186340892367426, + "train_runtime": 1737.4974, + "train_samples": 43598, + "train_samples_per_second": 25.092, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..cb8f87e --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.51007843017578, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.18576049804688, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.0026416778564453125, + "KL/mean": 0.0005560517311096191, + "KL/rejected_KL_mean": 0.0037555694580078125, + "KL/std": 0.2254001498222351, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.006394863128662109, + "fcm_dpo/q_t": 0.5001601576805115, + "grad_norm": 70.89234161376953, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4816657304763794, + "logits/rejected": -0.44219475984573364, + "logps/chosen": -60.98423767089844, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.66883850097656, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.006394565105438232, + "margin_dpo/margin_std": 0.3149183392524719, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.05694770812988281, + "KL/mean": -0.00047691166400909424, + "KL/rejected_KL_mean": 0.05599212646484375, + "KL/std": 0.27960479259490967, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.11293593049049377, + "fcm_dpo/q_t": 0.5028228759765625, + "grad_norm": 72.2734603881836, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.468903124332428, + "logits/rejected": -0.44123488664627075, + "logps/chosen": -56.82466125488281, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.59111785888672, + "loss": 1.3981, + "margin_dpo/margin_mean": -0.11293655633926392, + "margin_dpo/margin_std": 0.4282279312610626, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.006999969482421875, + "KL/mean": 0.010399580001831055, + "KL/rejected_KL_mean": 0.027801513671875, + "KL/std": 0.25716227293014526, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03479999303817749, + "fcm_dpo/q_t": 0.500869631767273, + "grad_norm": 89.93070220947266, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5143948793411255, + "logits/rejected": -0.4707018733024597, + "logps/chosen": -53.86637496948242, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.12138366699219, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.034800052642822266, + "margin_dpo/margin_std": 0.34687650203704834, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.07697486877441406, + "KL/mean": -0.042307183146476746, + "KL/rejected_KL_mean": -0.007640838623046875, + "KL/std": 0.2946726679801941, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06933179497718811, + "fcm_dpo/q_t": 0.5017322301864624, + "grad_norm": 92.47077941894531, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5228596329689026, + "logits/rejected": -0.4821878969669342, + "logps/chosen": -63.08445739746094, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.65298461914062, + "loss": 1.3937, + "margin_dpo/margin_mean": -0.06933215260505676, + "margin_dpo/margin_std": 0.4337119162082672, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.015333175659179688, + "KL/mean": 0.04118221998214722, + "KL/rejected_KL_mean": 0.0670318603515625, + "KL/std": 0.29051506519317627, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.051698148250579834, + "fcm_dpo/q_t": 0.5012915730476379, + "grad_norm": 82.6883544921875, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.501255989074707, + "logits/rejected": -0.4674297571182251, + "logps/chosen": -57.75948715209961, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.85356140136719, + "loss": 1.392, + "margin_dpo/margin_mean": -0.051698267459869385, + "margin_dpo/margin_std": 0.44515153765678406, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.05190086364746094, + "KL/mean": 0.04860445857048035, + "KL/rejected_KL_mean": 0.04531097412109375, + "KL/std": 0.3088279068470001, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0065915584564208984, + "fcm_dpo/q_t": 0.49983447790145874, + "grad_norm": 78.43338775634766, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5121935606002808, + "logits/rejected": -0.4866887032985687, + "logps/chosen": -58.66413497924805, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.26611328125, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.006591469049453735, + "margin_dpo/margin_std": 0.40233486890792847, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.0189208984375, + "KL/mean": -0.00609724223613739, + "KL/rejected_KL_mean": -0.0311126708984375, + "KL/std": 0.3066028654575348, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.050033897161483765, + "fcm_dpo/q_t": 0.498749315738678, + "grad_norm": 85.117919921875, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.5151736736297607, + "logits/rejected": -0.4697529673576355, + "logps/chosen": -69.8479232788086, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.63377380371094, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.050033748149871826, + "margin_dpo/margin_std": 0.4270361661911011, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.0309906005859375, + "KL/mean": 0.008808039128780365, + "KL/rejected_KL_mean": -0.013370513916015625, + "KL/std": 0.2620387375354767, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.044362738728523254, + "fcm_dpo/q_t": 0.49889153242111206, + "grad_norm": 70.67880249023438, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.4829540550708771, + "logits/rejected": -0.4391665458679199, + "logps/chosen": -48.32669448852539, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.38543701171875, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.0443631112575531, + "margin_dpo/margin_std": 0.3442138433456421, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.016119003295898438, + "KL/mean": 0.012044832110404968, + "KL/rejected_KL_mean": 0.00797271728515625, + "KL/std": 0.2124350368976593, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.008142799139022827, + "fcm_dpo/q_t": 0.49979639053344727, + "grad_norm": 68.35074615478516, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.46382224559783936, + "logits/rejected": -0.43896666169166565, + "logps/chosen": -53.000736236572266, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.77239990234375, + "loss": 1.3857, + "margin_dpo/margin_mean": 0.008143961429595947, + "margin_dpo/margin_std": 0.315945565700531, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.03598594665527344, + "KL/mean": -0.019736863672733307, + "KL/rejected_KL_mean": -0.00347900390625, + "KL/std": 0.29405325651168823, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03249834477901459, + "fcm_dpo/q_t": 0.5008106231689453, + "grad_norm": 96.05313110351562, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5169162750244141, + "logits/rejected": -0.48001742362976074, + "logps/chosen": -61.84142303466797, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.86175537109375, + "loss": 1.39, + "margin_dpo/margin_mean": -0.032499417662620544, + "margin_dpo/margin_std": 0.39334458112716675, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.008047103881835938, + "KL/mean": 0.02107308804988861, + "KL/rejected_KL_mean": 0.0341033935546875, + "KL/std": 0.30922242999076843, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.026049882173538208, + "fcm_dpo/q_t": 0.5006469488143921, + "grad_norm": 79.62260437011719, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.49720725417137146, + "logits/rejected": -0.4704585075378418, + "logps/chosen": -64.25230407714844, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.16897583007812, + "loss": 1.3895, + "margin_dpo/margin_mean": -0.02605000138282776, + "margin_dpo/margin_std": 0.4611724317073822, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.0075836181640625, + "KL/mean": -0.013656646013259888, + "KL/rejected_KL_mean": -0.0197296142578125, + "KL/std": 0.2594446539878845, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.012148141860961914, + "fcm_dpo/q_t": 0.49969619512557983, + "grad_norm": 85.17851257324219, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.46068429946899414, + "logits/rejected": -0.4207717776298523, + "logps/chosen": -58.117794036865234, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.06681060791016, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.012148261070251465, + "margin_dpo/margin_std": 0.4094587564468384, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.0186309814453125, + "KL/mean": -0.04213924705982208, + "KL/rejected_KL_mean": -0.06565093994140625, + "KL/std": 0.26634901762008667, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04701825976371765, + "fcm_dpo/q_t": 0.49882370233535767, + "grad_norm": 64.12860107421875, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.48477405309677124, + "logits/rejected": -0.46560412645339966, + "logps/chosen": -56.98554229736328, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.87429809570312, + "loss": 1.382, + "margin_dpo/margin_mean": 0.04701876640319824, + "margin_dpo/margin_std": 0.4151442348957062, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.02254486083984375, + "KL/mean": -0.044135212898254395, + "KL/rejected_KL_mean": -0.06572341918945312, + "KL/std": 0.263348788022995, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04317820072174072, + "fcm_dpo/q_t": 0.4989207983016968, + "grad_norm": 83.80663299560547, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5190131664276123, + "logits/rejected": -0.47741711139678955, + "logps/chosen": -61.76243591308594, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.43519592285156, + "loss": 1.3824, + "margin_dpo/margin_mean": 0.04317793250083923, + "margin_dpo/margin_std": 0.40978044271469116, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.045665740966796875, + "KL/mean": 0.01416228711605072, + "KL/rejected_KL_mean": -0.017337799072265625, + "KL/std": 0.2788323163986206, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0630052387714386, + "fcm_dpo/q_t": 0.4984249472618103, + "grad_norm": 78.6865005493164, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.49678856134414673, + "logits/rejected": -0.4590208828449249, + "logps/chosen": -67.6646728515625, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.39599609375, + "loss": 1.3804, + "margin_dpo/margin_mean": 0.06300631165504456, + "margin_dpo/margin_std": 0.3844602108001709, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.02127838134765625, + "KL/mean": -0.007436856627464294, + "KL/rejected_KL_mean": -0.03615570068359375, + "KL/std": 0.21797269582748413, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05743202567100525, + "fcm_dpo/q_t": 0.49856436252593994, + "grad_norm": 82.01636505126953, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.48908162117004395, + "logits/rejected": -0.4331856369972229, + "logps/chosen": -47.71821212768555, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.50843811035156, + "loss": 1.3808, + "margin_dpo/margin_mean": 0.05743253231048584, + "margin_dpo/margin_std": 0.30652177333831787, + "step": 18 + }, + { + "KL/chosen_KL_mean": -0.00011444091796875, + "KL/mean": -0.02399381995201111, + "KL/rejected_KL_mean": -0.0478668212890625, + "KL/std": 0.24001097679138184, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04774361848831177, + "fcm_dpo/q_t": 0.49880683422088623, + "grad_norm": 73.1715316772461, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.4793155789375305, + "logits/rejected": -0.42845872044563293, + "logps/chosen": -70.20547485351562, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.80545806884766, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.04774370789527893, + "margin_dpo/margin_std": 0.3320465683937073, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.0075206756591796875, + "KL/mean": -0.04018333554267883, + "KL/rejected_KL_mean": -0.07284164428710938, + "KL/std": 0.2719258666038513, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06532829999923706, + "fcm_dpo/q_t": 0.49836742877960205, + "grad_norm": 74.01097106933594, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.54817795753479, + "logits/rejected": -0.4915490746498108, + "logps/chosen": -50.810760498046875, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.89619445800781, + "loss": 1.3801, + "margin_dpo/margin_mean": 0.06532835960388184, + "margin_dpo/margin_std": 0.35930198431015015, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.009134292602539062, + "KL/mean": -0.04254424571990967, + "KL/rejected_KL_mean": -0.09421920776367188, + "KL/std": 0.2731036841869354, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10335588455200195, + "fcm_dpo/q_t": 0.4974174499511719, + "grad_norm": 77.20870208740234, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.5209602117538452, + "logits/rejected": -0.49909645318984985, + "logps/chosen": -50.05388259887695, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.9630126953125, + "loss": 1.3763, + "margin_dpo/margin_mean": 0.10335543751716614, + "margin_dpo/margin_std": 0.36786240339279175, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.016569137573242188, + "KL/mean": -0.06234532594680786, + "KL/rejected_KL_mean": -0.14126205444335938, + "KL/std": 0.28174692392349243, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1578305959701538, + "fcm_dpo/q_t": 0.49605605006217957, + "grad_norm": 84.69581604003906, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.4846010208129883, + "logits/rejected": -0.44085192680358887, + "logps/chosen": -59.04106903076172, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.64593505859375, + "loss": 1.3709, + "margin_dpo/margin_mean": 0.15783017873764038, + "margin_dpo/margin_std": 0.38242822885513306, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.05445098876953125, + "KL/mean": -0.06539598107337952, + "KL/rejected_KL_mean": -0.1852436065673828, + "KL/std": 0.3439909815788269, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23969513177871704, + "fcm_dpo/q_t": 0.4940136969089508, + "grad_norm": 80.23938751220703, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5010164380073547, + "logits/rejected": -0.4791816174983978, + "logps/chosen": -60.02324676513672, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.32479858398438, + "loss": 1.3631, + "margin_dpo/margin_mean": 0.2396949827671051, + "margin_dpo/margin_std": 0.4822561740875244, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.06022834777832031, + "KL/mean": -0.09138941019773483, + "KL/rejected_KL_mean": -0.243011474609375, + "KL/std": 0.288381963968277, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.30323976278305054, + "fcm_dpo/q_t": 0.49242281913757324, + "grad_norm": 85.59104919433594, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5155333876609802, + "logits/rejected": -0.49877071380615234, + "logps/chosen": -44.230804443359375, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.36822509765625, + "loss": 1.3566, + "margin_dpo/margin_mean": 0.30324065685272217, + "margin_dpo/margin_std": 0.3780192732810974, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.048633575439453125, + "KL/mean": -0.07320906221866608, + "KL/rejected_KL_mean": -0.1950531005859375, + "KL/std": 0.382267028093338, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2436859905719757, + "fcm_dpo/q_t": 0.49391207098960876, + "grad_norm": 73.8148193359375, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.51090407371521, + "logits/rejected": -0.4814668297767639, + "logps/chosen": -52.48841857910156, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.53724670410156, + "loss": 1.3627, + "margin_dpo/margin_mean": 0.2436855137348175, + "margin_dpo/margin_std": 0.477664589881897, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.11028099060058594, + "KL/mean": -0.09156893193721771, + "KL/rejected_KL_mean": -0.29341888427734375, + "KL/std": 0.4428122043609619, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4037046730518341, + "fcm_dpo/q_t": 0.489920437335968, + "grad_norm": 87.53692626953125, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.527031421661377, + "logits/rejected": -0.49516919255256653, + "logps/chosen": -53.81252670288086, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.65313720703125, + "loss": 1.3471, + "margin_dpo/margin_mean": 0.4037049412727356, + "margin_dpo/margin_std": 0.5555702447891235, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.08542823791503906, + "KL/mean": -0.14516480267047882, + "KL/rejected_KL_mean": -0.3757591247558594, + "KL/std": 0.4373488426208496, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4611866772174835, + "fcm_dpo/q_t": 0.48848414421081543, + "grad_norm": 93.67777252197266, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5126519799232483, + "logits/rejected": -0.47614771127700806, + "logps/chosen": -42.81310272216797, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.09996032714844, + "loss": 1.3415, + "margin_dpo/margin_mean": 0.4611864984035492, + "margin_dpo/margin_std": 0.5471404194831848, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.0010204315185546875, + "KL/mean": -0.1398795247077942, + "KL/rejected_KL_mean": -0.2807807922363281, + "KL/std": 0.40396660566329956, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.28180599212646484, + "fcm_dpo/q_t": 0.4929564297199249, + "grad_norm": 75.66232299804688, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5143972635269165, + "logits/rejected": -0.4599345922470093, + "logps/chosen": -60.55548095703125, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.68190002441406, + "loss": 1.3591, + "margin_dpo/margin_mean": 0.28180626034736633, + "margin_dpo/margin_std": 0.5587049722671509, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.11989593505859375, + "KL/mean": -0.1484879106283188, + "KL/rejected_KL_mean": -0.4168701171875, + "KL/std": 0.47871387004852295, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5367647409439087, + "fcm_dpo/q_t": 0.48659881949424744, + "grad_norm": 90.55548858642578, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5430953502655029, + "logits/rejected": -0.496090292930603, + "logps/chosen": -57.687889099121094, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.81121826171875, + "loss": 1.3342, + "margin_dpo/margin_mean": 0.5367645621299744, + "margin_dpo/margin_std": 0.5849047303199768, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.14161109924316406, + "KL/mean": -0.22270266711711884, + "KL/rejected_KL_mean": -0.5870208740234375, + "KL/std": 0.614432156085968, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7286314964294434, + "fcm_dpo/q_t": 0.4818177819252014, + "grad_norm": 87.20523834228516, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.4954381585121155, + "logits/rejected": -0.4648393392562866, + "logps/chosen": -52.435760498046875, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.07622528076172, + "loss": 1.3158, + "margin_dpo/margin_mean": 0.7286321520805359, + "margin_dpo/margin_std": 0.6483098268508911, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.10785293579101562, + "KL/mean": -0.14253471791744232, + "KL/rejected_KL_mean": -0.39292144775390625, + "KL/std": 0.5532994270324707, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5007778406143188, + "fcm_dpo/q_t": 0.4875016212463379, + "grad_norm": 68.83615112304688, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.4523066282272339, + "logits/rejected": -0.40141046047210693, + "logps/chosen": -63.699066162109375, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.28692626953125, + "loss": 1.3379, + "margin_dpo/margin_mean": 0.500777542591095, + "margin_dpo/margin_std": 0.6453180313110352, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.20159912109375, + "KL/mean": -0.1824570745229721, + "KL/rejected_KL_mean": -0.5665168762207031, + "KL/std": 0.6936391592025757, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7681108713150024, + "fcm_dpo/q_t": 0.4808514714241028, + "grad_norm": 82.05474090576172, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5182451605796814, + "logits/rejected": -0.47726649045944214, + "logps/chosen": -62.537925720214844, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.88401794433594, + "loss": 1.3126, + "margin_dpo/margin_mean": 0.7681113481521606, + "margin_dpo/margin_std": 0.8168035745620728, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.10602378845214844, + "KL/mean": -0.16686028242111206, + "KL/rejected_KL_mean": -0.4397392272949219, + "KL/std": 0.4986187219619751, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5457603335380554, + "fcm_dpo/q_t": 0.486371785402298, + "grad_norm": 72.55486297607422, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.5073941946029663, + "logits/rejected": -0.48181116580963135, + "logps/chosen": -53.15494918823242, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.32487487792969, + "loss": 1.3332, + "margin_dpo/margin_mean": 0.5457603931427002, + "margin_dpo/margin_std": 0.5410532355308533, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.08002853393554688, + "KL/mean": -0.3157731294631958, + "KL/rejected_KL_mean": -0.7115745544433594, + "KL/std": 0.7231928706169128, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7915985584259033, + "fcm_dpo/q_t": 0.48026302456855774, + "grad_norm": 78.1650619506836, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.4835454821586609, + "logits/rejected": -0.4660688042640686, + "logps/chosen": -50.73729705810547, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.6334228515625, + "loss": 1.3105, + "margin_dpo/margin_mean": 0.7915977835655212, + "margin_dpo/margin_std": 0.8456913232803345, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.1514110565185547, + "KL/mean": -0.4338723421096802, + "KL/rejected_KL_mean": -1.0191574096679688, + "KL/std": 1.0168572664260864, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1705669164657593, + "fcm_dpo/q_t": 0.47094202041625977, + "grad_norm": 81.89360809326172, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.508651614189148, + "logits/rejected": -0.471946656703949, + "logps/chosen": -50.873077392578125, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.84358215332031, + "loss": 1.2763, + "margin_dpo/margin_mean": 1.1705663204193115, + "margin_dpo/margin_std": 1.1983381509780884, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.04232215881347656, + "KL/mean": -0.5407265424728394, + "KL/rejected_KL_mean": -1.1237754821777344, + "KL/std": 1.1087287664413452, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.166100025177002, + "fcm_dpo/q_t": 0.47101762890815735, + "grad_norm": 72.48859405517578, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5650780200958252, + "logits/rejected": -0.5290362238883972, + "logps/chosen": -51.94916915893555, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.16439819335938, + "loss": 1.2773, + "margin_dpo/margin_mean": 1.1660990715026855, + "margin_dpo/margin_std": 1.262919306755066, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.009515762329101562, + "KL/mean": -0.5290871858596802, + "KL/rejected_KL_mean": -1.0486602783203125, + "KL/std": 1.0679088830947876, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0391461849212646, + "fcm_dpo/q_t": 0.474229097366333, + "grad_norm": 61.6435661315918, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5242244005203247, + "logits/rejected": -0.4816312789916992, + "logps/chosen": -62.81662368774414, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.94373321533203, + "loss": 1.2899, + "margin_dpo/margin_mean": 1.0391463041305542, + "margin_dpo/margin_std": 1.3453397750854492, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1499652862548828, + "KL/mean": -0.5511326193809509, + "KL/rejected_KL_mean": -1.2522315979003906, + "KL/std": 1.3804535865783691, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4021968841552734, + "fcm_dpo/q_t": 0.4654462933540344, + "grad_norm": 69.51306915283203, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5358166694641113, + "logits/rejected": -0.5031454563140869, + "logps/chosen": -48.24055480957031, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.1646728515625, + "loss": 1.2583, + "margin_dpo/margin_mean": 1.4021967649459839, + "margin_dpo/margin_std": 1.7107752561569214, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.09138107299804688, + "KL/mean": -0.7316733598709106, + "KL/rejected_KL_mean": -1.5547218322753906, + "KL/std": 1.338223934173584, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6461036205291748, + "fcm_dpo/q_t": 0.45919936895370483, + "grad_norm": 73.23494720458984, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5409366488456726, + "logits/rejected": -0.4999801516532898, + "logps/chosen": -50.65909194946289, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.12423706054688, + "loss": 1.2338, + "margin_dpo/margin_mean": 1.646103858947754, + "margin_dpo/margin_std": 1.4399521350860596, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.22576141357421875, + "KL/mean": -0.57262122631073, + "KL/rejected_KL_mean": -1.371002197265625, + "KL/std": 1.4182815551757812, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5967631340026855, + "fcm_dpo/q_t": 0.46059292554855347, + "grad_norm": 59.98262023925781, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.53311687707901, + "logits/rejected": -0.5040132999420166, + "logps/chosen": -57.759307861328125, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.67107391357422, + "loss": 1.2406, + "margin_dpo/margin_mean": 1.5967634916305542, + "margin_dpo/margin_std": 1.7029635906219482, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.057689666748046875, + "KL/mean": -0.9135010242462158, + "KL/rejected_KL_mean": -1.8846893310546875, + "KL/std": 1.837777018547058, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10297223925590515, + "fcm_dpo/delta": 0.09624151885509491, + "fcm_dpo/margin": 1.9423800706863403, + "fcm_dpo/q_t": 0.4522102475166321, + "grad_norm": 70.01563262939453, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5270588397979736, + "logits/rejected": -0.49000197649002075, + "logps/chosen": -62.638126373291016, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.90821838378906, + "loss": 1.2064, + "margin_dpo/margin_mean": 1.9423803091049194, + "margin_dpo/margin_std": 1.9404737949371338, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.22831344604492188, + "KL/mean": -1.0661954879760742, + "KL/rejected_KL_mean": -2.3607025146484375, + "KL/std": 2.158304214477539, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.11214305460453033, + "fcm_dpo/delta": 0.11957399547100067, + "fcm_dpo/margin": 2.5890145301818848, + "fcm_dpo/q_t": 0.43188583850860596, + "grad_norm": 86.09832763671875, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5553675889968872, + "logits/rejected": -0.5090369582176208, + "logps/chosen": -58.73811340332031, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.26907348632812, + "loss": 1.1347, + "margin_dpo/margin_mean": 2.5890140533447266, + "margin_dpo/margin_std": 2.432143449783325, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5291996002197266, + "KL/mean": -0.7373908758163452, + "KL/rejected_KL_mean": -2.003986358642578, + "KL/std": 1.8573217391967773, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.11914066970348358, + "fcm_dpo/delta": 0.10819369554519653, + "fcm_dpo/margin": 2.533182144165039, + "fcm_dpo/q_t": 0.4286477565765381, + "grad_norm": 81.7889175415039, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5308007597923279, + "logits/rejected": -0.5052840709686279, + "logps/chosen": -53.62679672241211, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.48417663574219, + "loss": 1.1207, + "margin_dpo/margin_mean": 2.533182144165039, + "margin_dpo/margin_std": 1.9596796035766602, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.19987106323242188, + "KL/mean": -1.1974523067474365, + "KL/rejected_KL_mean": -2.594776153564453, + "KL/std": 2.2577269077301025, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.1243496686220169, + "fcm_dpo/delta": 0.05869706720113754, + "fcm_dpo/margin": 2.7946481704711914, + "fcm_dpo/q_t": 0.41725268959999084, + "grad_norm": 92.5706787109375, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4627833366394043, + "logits/rejected": -0.4430672824382782, + "logps/chosen": -49.87862777709961, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.3785400390625, + "loss": 1.087, + "margin_dpo/margin_mean": 2.7946486473083496, + "margin_dpo/margin_std": 2.2088088989257812, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.12192344665527344, + "KL/mean": -1.0339019298553467, + "KL/rejected_KL_mean": -2.1897315979003906, + "KL/std": 2.084545850753784, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.13241304457187653, + "fcm_dpo/delta": 0.1025981456041336, + "fcm_dpo/margin": 2.3116536140441895, + "fcm_dpo/q_t": 0.4289446771144867, + "grad_norm": 75.87535858154297, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4669506549835205, + "logits/rejected": -0.4539613425731659, + "logps/chosen": -48.293006896972656, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.12615966796875, + "loss": 1.1313, + "margin_dpo/margin_mean": 2.3116531372070312, + "margin_dpo/margin_std": 2.569169044494629, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.20899009704589844, + "KL/mean": -1.267395257949829, + "KL/rejected_KL_mean": -2.7437820434570312, + "KL/std": 2.6592025756835938, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.13460655510425568, + "fcm_dpo/delta": 0.002703934907913208, + "fcm_dpo/margin": 2.9527716636657715, + "fcm_dpo/q_t": 0.4073982238769531, + "grad_norm": 85.6302719116211, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.537663459777832, + "logits/rejected": -0.48707181215286255, + "logps/chosen": -55.790435791015625, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.39636993408203, + "loss": 1.0714, + "margin_dpo/margin_mean": 2.9527716636657715, + "margin_dpo/margin_std": 3.2348241806030273, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.41410064697265625, + "KL/mean": -1.0927481651306152, + "KL/rejected_KL_mean": -2.5996017456054688, + "KL/std": 2.614330768585205, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.13522110879421234, + "fcm_dpo/delta": -0.009656481444835663, + "fcm_dpo/margin": 3.013702154159546, + "fcm_dpo/q_t": 0.4027646481990814, + "grad_norm": 80.39930725097656, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5810012221336365, + "logits/rejected": -0.528842031955719, + "logps/chosen": -57.51197814941406, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.27880859375, + "loss": 1.0531, + "margin_dpo/margin_mean": 3.013702154159546, + "margin_dpo/margin_std": 2.6816351413726807, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.056026458740234375, + "KL/mean": -1.5249731540679932, + "KL/rejected_KL_mean": -3.1059722900390625, + "KL/std": 2.708984613418579, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.1306951344013214, + "fcm_dpo/delta": -0.016385827213525772, + "fcm_dpo/margin": 3.1619958877563477, + "fcm_dpo/q_t": 0.40188631415367126, + "grad_norm": 84.90986633300781, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5903971791267395, + "logits/rejected": -0.5321975350379944, + "logps/chosen": -57.132049560546875, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.12257385253906, + "loss": 1.057, + "margin_dpo/margin_mean": 3.1619958877563477, + "margin_dpo/margin_std": 2.970660924911499, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.2901954650878906, + "KL/mean": -1.60369873046875, + "KL/rejected_KL_mean": -3.4975929260253906, + "KL/std": 3.445910692214966, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.1277078539133072, + "fcm_dpo/delta": -0.09768079966306686, + "fcm_dpo/margin": 3.7877845764160156, + "fcm_dpo/q_t": 0.38733533024787903, + "grad_norm": 78.39299011230469, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5638530254364014, + "logits/rejected": -0.5069452524185181, + "logps/chosen": -61.39507293701172, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.26506805419922, + "loss": 1.0197, + "margin_dpo/margin_mean": 3.7877838611602783, + "margin_dpo/margin_std": 3.9767532348632812, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.1568470001220703, + "KL/mean": -2.231832504272461, + "KL/rejected_KL_mean": -4.306819915771484, + "KL/std": 3.528477191925049, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.11805199086666107, + "fcm_dpo/delta": -0.10631946474313736, + "fcm_dpo/margin": 4.149969577789307, + "fcm_dpo/q_t": 0.38349735736846924, + "grad_norm": 69.52253723144531, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5696861743927002, + "logits/rejected": -0.5346043109893799, + "logps/chosen": -58.880985260009766, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.66496276855469, + "loss": 1.0066, + "margin_dpo/margin_mean": 4.149970054626465, + "margin_dpo/margin_std": 4.02549409866333, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2366199493408203, + "KL/mean": -2.2376832962036133, + "KL/rejected_KL_mean": -4.238750457763672, + "KL/std": 4.155153751373291, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.11174145340919495, + "fcm_dpo/delta": -0.05799061432480812, + "fcm_dpo/margin": 4.002124786376953, + "fcm_dpo/q_t": 0.3985278308391571, + "grad_norm": 56.468875885009766, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5277886390686035, + "logits/rejected": -0.495082825422287, + "logps/chosen": -61.61029052734375, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.24075317382812, + "loss": 1.0624, + "margin_dpo/margin_mean": 4.002124786376953, + "margin_dpo/margin_std": 5.097770690917969, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.38054656982421875, + "KL/mean": -2.713926315307617, + "KL/rejected_KL_mean": -5.808399200439453, + "KL/std": 4.801059722900391, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.09976285696029663, + "fcm_dpo/delta": -0.2687903642654419, + "fcm_dpo/margin": 6.188947677612305, + "fcm_dpo/q_t": 0.35018402338027954, + "grad_norm": 59.128517150878906, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5515159368515015, + "logits/rejected": -0.4958212971687317, + "logps/chosen": -51.956809997558594, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.78231811523438, + "loss": 0.92, + "margin_dpo/margin_mean": 6.188947677612305, + "margin_dpo/margin_std": 5.1786909103393555, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.18945693969726562, + "KL/mean": -3.3026957511901855, + "KL/rejected_KL_mean": -6.415935516357422, + "KL/std": 5.387790679931641, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.09013447165489197, + "fcm_dpo/delta": -0.1929309070110321, + "fcm_dpo/margin": 6.2264814376831055, + "fcm_dpo/q_t": 0.3713955283164978, + "grad_norm": 56.34618377685547, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.5957865118980408, + "logits/rejected": -0.5736597776412964, + "logps/chosen": -53.50410842895508, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.19953155517578, + "loss": 0.9794, + "margin_dpo/margin_mean": 6.2264814376831055, + "margin_dpo/margin_std": 6.150714874267578, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3715839385986328, + "KL/mean": -3.0501108169555664, + "KL/rejected_KL_mean": -5.728633880615234, + "KL/std": 5.062835216522217, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.0837288424372673, + "fcm_dpo/delta": -0.05597742646932602, + "fcm_dpo/margin": 5.357050895690918, + "fcm_dpo/q_t": 0.3939096927642822, + "grad_norm": 63.238712310791016, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5826476812362671, + "logits/rejected": -0.5287748575210571, + "logps/chosen": -51.060245513916016, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.44403076171875, + "loss": 1.029, + "margin_dpo/margin_mean": 5.357049942016602, + "margin_dpo/margin_std": 5.098629474639893, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9310493469238281, + "KL/mean": -4.164062023162842, + "KL/rejected_KL_mean": -7.397075653076172, + "KL/std": 6.21592903137207, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.07877273857593536, + "fcm_dpo/delta": -0.12850306928157806, + "fcm_dpo/margin": 6.46602725982666, + "fcm_dpo/q_t": 0.3825332224369049, + "grad_norm": 46.757259368896484, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6221465468406677, + "logits/rejected": -0.5574856996536255, + "logps/chosen": -63.54628372192383, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.39057922363281, + "loss": 1.0203, + "margin_dpo/margin_mean": 6.46602725982666, + "margin_dpo/margin_std": 7.513976097106934, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.6751270294189453, + "KL/mean": -3.748490571975708, + "KL/rejected_KL_mean": -6.821861267089844, + "KL/std": 5.988111972808838, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.0744306892156601, + "fcm_dpo/delta": -0.06656648218631744, + "fcm_dpo/margin": 6.146733283996582, + "fcm_dpo/q_t": 0.39414769411087036, + "grad_norm": 41.26564025878906, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.5941236019134521, + "logits/rejected": -0.5509048104286194, + "logps/chosen": -58.60785675048828, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -100.99629974365234, + "loss": 1.0476, + "margin_dpo/margin_mean": 6.146733283996582, + "margin_dpo/margin_std": 7.233534812927246, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7006874084472656, + "KL/mean": -4.188282012939453, + "KL/rejected_KL_mean": -7.675880432128906, + "KL/std": 5.985398292541504, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.07077182829380035, + "fcm_dpo/delta": -0.10953762382268906, + "fcm_dpo/margin": 6.975188255310059, + "fcm_dpo/q_t": 0.38194090127944946, + "grad_norm": 45.177852630615234, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5877989530563354, + "logits/rejected": -0.5601094961166382, + "logps/chosen": -71.19596862792969, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.2413330078125, + "loss": 1.0046, + "margin_dpo/margin_mean": 6.975189208984375, + "margin_dpo/margin_std": 6.579164505004883, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.0104236602783203, + "KL/mean": -4.830958366394043, + "KL/rejected_KL_mean": -8.651496887207031, + "KL/std": 6.762295246124268, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.06591323018074036, + "fcm_dpo/delta": -0.12158824503421783, + "fcm_dpo/margin": 7.641064643859863, + "fcm_dpo/q_t": 0.38219478726387024, + "grad_norm": 45.31306457519531, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.5917017459869385, + "logits/rejected": -0.5152237415313721, + "logps/chosen": -63.14336395263672, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.26879119873047, + "loss": 1.0105, + "margin_dpo/margin_mean": 7.64106559753418, + "margin_dpo/margin_std": 8.053571701049805, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.5020942687988281, + "KL/mean": -5.6870927810668945, + "KL/rejected_KL_mean": -9.872089385986328, + "KL/std": 7.387317180633545, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.05950796976685524, + "fcm_dpo/delta": -0.1205754429101944, + "fcm_dpo/margin": 8.369996070861816, + "fcm_dpo/q_t": 0.3818981945514679, + "grad_norm": 42.901371002197266, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6331356167793274, + "logits/rejected": -0.5932568311691284, + "logps/chosen": -53.43461990356445, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.75729370117188, + "loss": 1.0089, + "margin_dpo/margin_mean": 8.369997024536133, + "margin_dpo/margin_std": 8.489030838012695, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.646495819091797, + "KL/mean": -6.035104751586914, + "KL/rejected_KL_mean": -9.423709869384766, + "KL/std": 6.781810760498047, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.05906199663877487, + "fcm_dpo/delta": -0.0007294043898582458, + "fcm_dpo/margin": 6.777213096618652, + "fcm_dpo/q_t": 0.4053270220756531, + "grad_norm": 43.15125274658203, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6143454313278198, + "logits/rejected": -0.554495096206665, + "logps/chosen": -63.58868408203125, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.81710815429688, + "loss": 1.0751, + "margin_dpo/margin_mean": 6.777214050292969, + "margin_dpo/margin_std": 7.661109924316406, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.4944896697998047, + "KL/mean": -5.693574905395508, + "KL/rejected_KL_mean": -9.892662048339844, + "KL/std": 9.201791763305664, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.056633442640304565, + "fcm_dpo/delta": -0.08983364701271057, + "fcm_dpo/margin": 8.39816951751709, + "fcm_dpo/q_t": 0.3943619728088379, + "grad_norm": 37.644508361816406, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6178034543991089, + "logits/rejected": -0.5840914249420166, + "logps/chosen": -62.128013610839844, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.74516296386719, + "loss": 1.0533, + "margin_dpo/margin_mean": 8.39816951751709, + "margin_dpo/margin_std": 11.410362243652344, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.7324161529541016, + "KL/mean": -4.843903541564941, + "KL/rejected_KL_mean": -7.955387115478516, + "KL/std": 6.692473411560059, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.05799160897731781, + "fcm_dpo/delta": 0.04286958649754524, + "fcm_dpo/margin": 6.222973346710205, + "fcm_dpo/q_t": 0.41731971502304077, + "grad_norm": 36.23487091064453, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.598295271396637, + "logits/rejected": -0.563392162322998, + "logps/chosen": -57.88318634033203, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.5215835571289, + "loss": 1.1119, + "margin_dpo/margin_mean": 6.222973346710205, + "margin_dpo/margin_std": 8.206045150756836, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.807455062866211, + "KL/mean": -7.064422607421875, + "KL/rejected_KL_mean": -11.32138442993164, + "KL/std": 8.585464477539062, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.05498577654361725, + "fcm_dpo/delta": -0.08206503093242645, + "fcm_dpo/margin": 8.513931274414062, + "fcm_dpo/q_t": 0.3888593316078186, + "grad_norm": 39.08759689331055, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.5942816734313965, + "logits/rejected": -0.5492488145828247, + "logps/chosen": -75.95484924316406, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -108.93144226074219, + "loss": 1.0306, + "margin_dpo/margin_mean": 8.513931274414062, + "margin_dpo/margin_std": 9.029535293579102, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.1175975799560547, + "KL/mean": -6.1625871658325195, + "KL/rejected_KL_mean": -11.207576751708984, + "KL/std": 9.273113250732422, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.050512149930000305, + "fcm_dpo/delta": -0.1379590928554535, + "fcm_dpo/margin": 10.08997631072998, + "fcm_dpo/q_t": 0.3805599808692932, + "grad_norm": 35.37334060668945, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5967783331871033, + "logits/rejected": -0.5650485754013062, + "logps/chosen": -55.11619567871094, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.73777770996094, + "loss": 1.0158, + "margin_dpo/margin_mean": 10.08997631072998, + "margin_dpo/margin_std": 11.369918823242188, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.4854373931884766, + "KL/mean": -8.625164031982422, + "KL/rejected_KL_mean": -13.764892578125, + "KL/std": 10.580974578857422, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.047643546015024185, + "fcm_dpo/delta": -0.10833816975355148, + "fcm_dpo/margin": 10.27945613861084, + "fcm_dpo/q_t": 0.38398468494415283, + "grad_norm": 35.62152099609375, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6589208841323853, + "logits/rejected": -0.6464786529541016, + "logps/chosen": -68.32144165039062, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.71134948730469, + "loss": 1.0271, + "margin_dpo/margin_mean": 10.27945613861084, + "margin_dpo/margin_std": 11.594311714172363, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.2952136993408203, + "KL/mean": -7.96829080581665, + "KL/rejected_KL_mean": -12.641365051269531, + "KL/std": 9.491405487060547, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.04598717764019966, + "fcm_dpo/delta": -0.036736130714416504, + "fcm_dpo/margin": 9.34615421295166, + "fcm_dpo/q_t": 0.4011841416358948, + "grad_norm": 31.296119689941406, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6361862421035767, + "logits/rejected": -0.6030631065368652, + "logps/chosen": -54.738739013671875, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -88.27766418457031, + "loss": 1.0653, + "margin_dpo/margin_mean": 9.34615421295166, + "margin_dpo/margin_std": 11.400927543640137, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.8825111389160156, + "KL/mean": -7.737419128417969, + "KL/rejected_KL_mean": -12.592334747314453, + "KL/std": 9.7019681930542, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.04574979096651077, + "fcm_dpo/delta": -0.05100099742412567, + "fcm_dpo/margin": 9.709821701049805, + "fcm_dpo/q_t": 0.398111492395401, + "grad_norm": 32.484439849853516, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.5914376974105835, + "logits/rejected": -0.5475878715515137, + "logps/chosen": -62.22331619262695, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -85.37962341308594, + "loss": 1.0554, + "margin_dpo/margin_mean": 9.709820747375488, + "margin_dpo/margin_std": 11.745061874389648, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.6033401489257812, + "KL/mean": -8.22460651397705, + "KL/rejected_KL_mean": -12.845874786376953, + "KL/std": 8.6856689453125, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.04498521238565445, + "fcm_dpo/delta": -0.018180008977651596, + "fcm_dpo/margin": 9.242530822753906, + "fcm_dpo/q_t": 0.40245479345321655, + "grad_norm": 31.0687313079834, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6392113566398621, + "logits/rejected": -0.5777957439422607, + "logps/chosen": -68.80917358398438, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -90.0531234741211, + "loss": 1.0581, + "margin_dpo/margin_mean": 9.242530822753906, + "margin_dpo/margin_std": 9.70833969116211, + "step": 68 + }, + { + "KL/chosen_KL_mean": -4.411651611328125, + "KL/mean": -10.337658882141113, + "KL/rejected_KL_mean": -16.263668060302734, + "KL/std": 10.823379516601562, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.04184731841087341, + "fcm_dpo/delta": -0.11539403349161148, + "fcm_dpo/margin": 11.852014541625977, + "fcm_dpo/q_t": 0.3833322525024414, + "grad_norm": 33.28119659423828, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6251046657562256, + "logits/rejected": -0.6021096706390381, + "logps/chosen": -64.23089599609375, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -119.65252685546875, + "loss": 1.0074, + "margin_dpo/margin_mean": 11.852014541625977, + "margin_dpo/margin_std": 11.963472366333008, + "step": 69 + }, + { + "KL/chosen_KL_mean": -6.616657257080078, + "KL/mean": -13.047698020935059, + "KL/rejected_KL_mean": -19.478740692138672, + "KL/std": 13.716583251953125, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.03962002322077751, + "fcm_dpo/delta": -0.1288178414106369, + "fcm_dpo/margin": 12.862080574035645, + "fcm_dpo/q_t": 0.382204532623291, + "grad_norm": 33.18239974975586, + "learning_rate": 5e-07, + "logits/chosen": -0.6271393299102783, + "logits/rejected": -0.5930898189544678, + "logps/chosen": -68.54730224609375, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -110.53952026367188, + "loss": 1.0172, + "margin_dpo/margin_mean": 12.862081527709961, + "margin_dpo/margin_std": 14.6132173538208, + "step": 70 + }, + { + "KL/chosen_KL_mean": -6.653339385986328, + "KL/mean": -14.409233093261719, + "KL/rejected_KL_mean": -22.165130615234375, + "KL/std": 14.737483978271484, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.03520714119076729, + "fcm_dpo/delta": -0.1779247373342514, + "fcm_dpo/margin": 15.511784553527832, + "fcm_dpo/q_t": 0.3739624321460724, + "grad_norm": 30.8501033782959, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6573929786682129, + "logits/rejected": -0.619554877281189, + "logps/chosen": -68.40367889404297, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -119.50175476074219, + "loss": 0.9864, + "margin_dpo/margin_mean": 15.511785507202148, + "margin_dpo/margin_std": 16.170812606811523, + "step": 71 + }, + { + "KL/chosen_KL_mean": -7.775922775268555, + "KL/mean": -15.911123275756836, + "KL/rejected_KL_mean": -24.04632568359375, + "KL/std": 15.989891052246094, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.03222007304430008, + "fcm_dpo/delta": -0.148702010512352, + "fcm_dpo/margin": 16.270404815673828, + "fcm_dpo/q_t": 0.37796416878700256, + "grad_norm": 28.37940216064453, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6923967599868774, + "logits/rejected": -0.65821373462677, + "logps/chosen": -73.82933044433594, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -119.33331298828125, + "loss": 1.0111, + "margin_dpo/margin_mean": 16.270404815673828, + "margin_dpo/margin_std": 18.938623428344727, + "step": 72 + }, + { + "KL/chosen_KL_mean": -9.934659957885742, + "KL/mean": -18.146343231201172, + "KL/rejected_KL_mean": -26.358020782470703, + "KL/std": 20.785381317138672, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.029831619933247566, + "fcm_dpo/delta": -0.10776865482330322, + "fcm_dpo/margin": 16.42336082458496, + "fcm_dpo/q_t": 0.39411330223083496, + "grad_norm": 27.882951736450195, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6926304697990417, + "logits/rejected": -0.6529533267021179, + "logps/chosen": -76.19093322753906, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -116.81415557861328, + "loss": 1.0862, + "margin_dpo/margin_mean": 16.423358917236328, + "margin_dpo/margin_std": 26.16392707824707, + "step": 73 + }, + { + "KL/chosen_KL_mean": -10.472402572631836, + "KL/mean": -19.609771728515625, + "KL/rejected_KL_mean": -28.74713897705078, + "KL/std": 23.029720306396484, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.02784036472439766, + "fcm_dpo/delta": -0.129630446434021, + "fcm_dpo/margin": 18.274738311767578, + "fcm_dpo/q_t": 0.39063596725463867, + "grad_norm": 28.453807830810547, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.658713698387146, + "logits/rejected": -0.6525105237960815, + "logps/chosen": -63.89728546142578, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -124.69407653808594, + "loss": 1.0595, + "margin_dpo/margin_mean": 18.274738311767578, + "margin_dpo/margin_std": 28.87874984741211, + "step": 74 + }, + { + "KL/chosen_KL_mean": -10.320928573608398, + "KL/mean": -23.239727020263672, + "KL/rejected_KL_mean": -36.15851593017578, + "KL/std": 23.612171173095703, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.024590618908405304, + "fcm_dpo/delta": -0.28821027278900146, + "fcm_dpo/margin": 25.837600708007812, + "fcm_dpo/q_t": 0.35423314571380615, + "grad_norm": 26.047040939331055, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.673154890537262, + "logits/rejected": -0.6671382188796997, + "logps/chosen": -62.18259048461914, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -147.41250610351562, + "loss": 0.9475, + "margin_dpo/margin_mean": 25.837596893310547, + "margin_dpo/margin_std": 27.000972747802734, + "step": 75 + }, + { + "KL/chosen_KL_mean": -12.577730178833008, + "KL/mean": -20.559160232543945, + "KL/rejected_KL_mean": -28.540592193603516, + "KL/std": 19.1776065826416, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.023455604910850525, + "fcm_dpo/delta": 0.028665322810411453, + "fcm_dpo/margin": 15.962867736816406, + "fcm_dpo/q_t": 0.4126405119895935, + "grad_norm": 24.884567260742188, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.7083161473274231, + "logits/rejected": -0.6686010360717773, + "logps/chosen": -65.84376525878906, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -106.75721740722656, + "loss": 1.104, + "margin_dpo/margin_mean": 15.96286678314209, + "margin_dpo/margin_std": 21.119997024536133, + "step": 76 + }, + { + "KL/chosen_KL_mean": -11.378250122070312, + "KL/mean": -24.624454498291016, + "KL/rejected_KL_mean": -37.87066650390625, + "KL/std": 26.15625762939453, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.02117023803293705, + "fcm_dpo/delta": -0.1990165412425995, + "fcm_dpo/margin": 26.492416381835938, + "fcm_dpo/q_t": 0.36689239740371704, + "grad_norm": 25.755342483520508, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6823813915252686, + "logits/rejected": -0.6459943056106567, + "logps/chosen": -69.47492980957031, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -131.644287109375, + "loss": 0.989, + "margin_dpo/margin_mean": 26.492416381835938, + "margin_dpo/margin_std": 29.323362350463867, + "step": 77 + }, + { + "KL/chosen_KL_mean": -11.63412094116211, + "KL/mean": -21.414306640625, + "KL/rejected_KL_mean": -31.194488525390625, + "KL/std": 21.385738372802734, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.020893137902021408, + "fcm_dpo/delta": -0.010137060657143593, + "fcm_dpo/margin": 19.560367584228516, + "fcm_dpo/q_t": 0.4062430262565613, + "grad_norm": 22.976896286010742, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6544197797775269, + "logits/rejected": -0.6318216323852539, + "logps/chosen": -67.24790954589844, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -116.12885284423828, + "loss": 1.0824, + "margin_dpo/margin_mean": 19.560367584228516, + "margin_dpo/margin_std": 24.80806541442871, + "step": 78 + }, + { + "KL/chosen_KL_mean": -12.215566635131836, + "KL/mean": -22.913497924804688, + "KL/rejected_KL_mean": -33.61143112182617, + "KL/std": 23.972692489624023, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.02033204957842827, + "fcm_dpo/delta": -0.040337756276130676, + "fcm_dpo/margin": 21.395862579345703, + "fcm_dpo/q_t": 0.40213775634765625, + "grad_norm": 21.764850616455078, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6702351570129395, + "logits/rejected": -0.6428369283676147, + "logps/chosen": -67.66605377197266, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -121.25899505615234, + "loss": 1.0789, + "margin_dpo/margin_mean": 21.395862579345703, + "margin_dpo/margin_std": 29.323734283447266, + "step": 79 + }, + { + "KL/chosen_KL_mean": -15.706001281738281, + "KL/mean": -26.025676727294922, + "KL/rejected_KL_mean": -36.34535217285156, + "KL/std": 26.170520782470703, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.0197686105966568, + "fcm_dpo/delta": -0.01160670816898346, + "fcm_dpo/margin": 20.639347076416016, + "fcm_dpo/q_t": 0.410672664642334, + "grad_norm": 22.324342727661133, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.725035548210144, + "logits/rejected": -0.6865429878234863, + "logps/chosen": -74.22529602050781, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -123.89285278320312, + "loss": 1.1066, + "margin_dpo/margin_mean": 20.63934898376465, + "margin_dpo/margin_std": 30.568384170532227, + "step": 80 + }, + { + "KL/chosen_KL_mean": -15.05624008178711, + "KL/mean": -29.24221420288086, + "KL/rejected_KL_mean": -43.428192138671875, + "KL/std": 31.100942611694336, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.01850263774394989, + "fcm_dpo/delta": -0.1524014174938202, + "fcm_dpo/margin": 28.371946334838867, + "fcm_dpo/q_t": 0.38178181648254395, + "grad_norm": 23.19623565673828, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7049325108528137, + "logits/rejected": -0.6785398721694946, + "logps/chosen": -81.50511169433594, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -173.0908966064453, + "loss": 1.0287, + "margin_dpo/margin_mean": 28.371944427490234, + "margin_dpo/margin_std": 37.10132598876953, + "step": 81 + }, + { + "KL/chosen_KL_mean": -17.63132667541504, + "KL/mean": -28.930374145507812, + "KL/rejected_KL_mean": -40.229427337646484, + "KL/std": 28.514408111572266, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.018600784242153168, + "fcm_dpo/delta": -0.027498170733451843, + "fcm_dpo/margin": 22.598100662231445, + "fcm_dpo/q_t": 0.4082872271537781, + "grad_norm": 24.770950317382812, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6457772254943848, + "logits/rejected": -0.6376634836196899, + "logps/chosen": -69.86370849609375, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -130.97268676757812, + "loss": 1.1357, + "margin_dpo/margin_mean": 22.598100662231445, + "margin_dpo/margin_std": 39.59211730957031, + "step": 82 + }, + { + "KL/chosen_KL_mean": -18.2735652923584, + "KL/mean": -31.97967529296875, + "KL/rejected_KL_mean": -45.685794830322266, + "KL/std": 29.78600311279297, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.01720629259943962, + "fcm_dpo/delta": -0.08493149280548096, + "fcm_dpo/margin": 27.412220001220703, + "fcm_dpo/q_t": 0.3923688530921936, + "grad_norm": 22.08080291748047, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6751551628112793, + "logits/rejected": -0.6464759111404419, + "logps/chosen": -74.1009521484375, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -149.40170288085938, + "loss": 1.0548, + "margin_dpo/margin_mean": 27.412220001220703, + "margin_dpo/margin_std": 35.628173828125, + "step": 83 + }, + { + "KL/chosen_KL_mean": -17.693445205688477, + "KL/mean": -28.92867088317871, + "KL/rejected_KL_mean": -40.16389465332031, + "KL/std": 25.48638916015625, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.016952861100435257, + "fcm_dpo/delta": 0.019233372062444687, + "fcm_dpo/margin": 22.470455169677734, + "fcm_dpo/q_t": 0.41033631563186646, + "grad_norm": 21.044750213623047, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6786582469940186, + "logits/rejected": -0.6426678895950317, + "logps/chosen": -84.86961364746094, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -127.46249389648438, + "loss": 1.0916, + "margin_dpo/margin_mean": 22.470455169677734, + "margin_dpo/margin_std": 25.842065811157227, + "step": 84 + }, + { + "KL/chosen_KL_mean": -17.898305892944336, + "KL/mean": -29.70880889892578, + "KL/rejected_KL_mean": -41.51930618286133, + "KL/std": 26.055973052978516, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.0175352580845356, + "fcm_dpo/delta": -0.018575016409158707, + "fcm_dpo/margin": 23.621002197265625, + "fcm_dpo/q_t": 0.40505295991897583, + "grad_norm": 21.607826232910156, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6370229125022888, + "logits/rejected": -0.6043232679367065, + "logps/chosen": -76.30492401123047, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -120.15811157226562, + "loss": 1.0902, + "margin_dpo/margin_mean": 23.621002197265625, + "margin_dpo/margin_std": 31.47956085205078, + "step": 85 + }, + { + "KL/chosen_KL_mean": -23.537614822387695, + "KL/mean": -36.072853088378906, + "KL/rejected_KL_mean": -48.60809326171875, + "KL/std": 36.59751510620117, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.01709899678826332, + "fcm_dpo/delta": -0.03356310352683067, + "fcm_dpo/margin": 25.070476531982422, + "fcm_dpo/q_t": 0.4138457775115967, + "grad_norm": 25.600481033325195, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6667304039001465, + "logits/rejected": -0.6572592258453369, + "logps/chosen": -79.67507934570312, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -136.729736328125, + "loss": 1.1625, + "margin_dpo/margin_mean": 25.07047462463379, + "margin_dpo/margin_std": 51.08125305175781, + "step": 86 + }, + { + "KL/chosen_KL_mean": -22.95316505432129, + "KL/mean": -38.12034606933594, + "KL/rejected_KL_mean": -53.28752517700195, + "KL/std": 36.39125442504883, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.015853099524974823, + "fcm_dpo/delta": -0.09766344726085663, + "fcm_dpo/margin": 30.334362030029297, + "fcm_dpo/q_t": 0.3906514048576355, + "grad_norm": 21.555530548095703, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7122213840484619, + "logits/rejected": -0.6932553052902222, + "logps/chosen": -78.58926391601562, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -148.75509643554688, + "loss": 1.0783, + "margin_dpo/margin_mean": 30.334360122680664, + "margin_dpo/margin_std": 44.74089813232422, + "step": 87 + }, + { + "KL/chosen_KL_mean": -29.247955322265625, + "KL/mean": -40.70698547363281, + "KL/rejected_KL_mean": -52.166019439697266, + "KL/std": 38.763587951660156, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.015893250703811646, + "fcm_dpo/delta": 0.03924756869673729, + "fcm_dpo/margin": 22.91806411743164, + "fcm_dpo/q_t": 0.4218425154685974, + "grad_norm": 22.637527465820312, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6560019850730896, + "logits/rejected": -0.6534268856048584, + "logps/chosen": -102.91910552978516, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -158.87451171875, + "loss": 1.1665, + "margin_dpo/margin_mean": 22.91806411743164, + "margin_dpo/margin_std": 41.653865814208984, + "step": 88 + }, + { + "KL/chosen_KL_mean": -18.678335189819336, + "KL/mean": -31.60675811767578, + "KL/rejected_KL_mean": -44.535186767578125, + "KL/std": 32.93218994140625, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.016255218535661697, + "fcm_dpo/delta": -0.024069301784038544, + "fcm_dpo/margin": 25.856843948364258, + "fcm_dpo/q_t": 0.40495485067367554, + "grad_norm": 21.459228515625, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.6885330677032471, + "logits/rejected": -0.6579680442810059, + "logps/chosen": -79.30325317382812, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -126.61873626708984, + "loss": 1.0957, + "margin_dpo/margin_mean": 25.856842041015625, + "margin_dpo/margin_std": 38.22654724121094, + "step": 89 + }, + { + "KL/chosen_KL_mean": -22.90645408630371, + "KL/mean": -41.31105422973633, + "KL/rejected_KL_mean": -59.71564865112305, + "KL/std": 44.8308219909668, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.015142069198191166, + "fcm_dpo/delta": -0.18906158208847046, + "fcm_dpo/margin": 36.80919647216797, + "fcm_dpo/q_t": 0.38302797079086304, + "grad_norm": 21.79202651977539, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6273738741874695, + "logits/rejected": -0.6466231346130371, + "logps/chosen": -76.19176483154297, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -171.26036071777344, + "loss": 1.066, + "margin_dpo/margin_mean": 36.80919647216797, + "margin_dpo/margin_std": 56.07073974609375, + "step": 90 + }, + { + "KL/chosen_KL_mean": -24.558679580688477, + "KL/mean": -38.63119888305664, + "KL/rejected_KL_mean": -52.70372009277344, + "KL/std": 35.13978576660156, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.014363479800522327, + "fcm_dpo/delta": -0.005329811945557594, + "fcm_dpo/margin": 28.145034790039062, + "fcm_dpo/q_t": 0.41156575083732605, + "grad_norm": 20.488649368286133, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6677622199058533, + "logits/rejected": -0.6377185583114624, + "logps/chosen": -86.36163330078125, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -140.57766723632812, + "loss": 1.1241, + "margin_dpo/margin_mean": 28.145034790039062, + "margin_dpo/margin_std": 46.73609924316406, + "step": 91 + }, + { + "KL/chosen_KL_mean": -21.91731834411621, + "KL/mean": -36.504920959472656, + "KL/rejected_KL_mean": -51.092525482177734, + "KL/std": 36.32714080810547, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.014314822852611542, + "fcm_dpo/delta": -0.0215253084897995, + "fcm_dpo/margin": 29.175203323364258, + "fcm_dpo/q_t": 0.4069999158382416, + "grad_norm": 19.61992073059082, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.6793630123138428, + "logits/rejected": -0.6574571132659912, + "logps/chosen": -73.55809020996094, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -128.9737091064453, + "loss": 1.1043, + "margin_dpo/margin_mean": 29.175203323364258, + "margin_dpo/margin_std": 44.00428009033203, + "step": 92 + }, + { + "KL/chosen_KL_mean": -24.591529846191406, + "KL/mean": -39.12522888183594, + "KL/rejected_KL_mean": -53.65892791748047, + "KL/std": 33.62416076660156, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.014067547395825386, + "fcm_dpo/delta": -0.01035611517727375, + "fcm_dpo/margin": 29.067394256591797, + "fcm_dpo/q_t": 0.4076431691646576, + "grad_norm": 20.400930404663086, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7195478677749634, + "logits/rejected": -0.677873432636261, + "logps/chosen": -77.12077331542969, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -130.81967163085938, + "loss": 1.0879, + "margin_dpo/margin_mean": 29.067394256591797, + "margin_dpo/margin_std": 39.640655517578125, + "step": 93 + }, + { + "KL/chosen_KL_mean": -26.388696670532227, + "KL/mean": -43.5217399597168, + "KL/rejected_KL_mean": -60.654788970947266, + "KL/std": 39.52867889404297, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.013327440246939659, + "fcm_dpo/delta": -0.06864941865205765, + "fcm_dpo/margin": 34.266090393066406, + "fcm_dpo/q_t": 0.39485517144203186, + "grad_norm": 19.593334197998047, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6383334398269653, + "logits/rejected": -0.6286529898643494, + "logps/chosen": -87.61131286621094, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -160.25381469726562, + "loss": 1.0542, + "margin_dpo/margin_mean": 34.266090393066406, + "margin_dpo/margin_std": 42.075103759765625, + "step": 94 + }, + { + "KL/chosen_KL_mean": -25.744752883911133, + "KL/mean": -39.528175354003906, + "KL/rejected_KL_mean": -53.311607360839844, + "KL/std": 39.386260986328125, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.013443037867546082, + "fcm_dpo/delta": 0.0326070636510849, + "fcm_dpo/margin": 27.566856384277344, + "fcm_dpo/q_t": 0.41484612226486206, + "grad_norm": 19.52400016784668, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6795843243598938, + "logits/rejected": -0.6558481454849243, + "logps/chosen": -78.26840209960938, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -129.19195556640625, + "loss": 1.1232, + "margin_dpo/margin_mean": 27.566856384277344, + "margin_dpo/margin_std": 41.232208251953125, + "step": 95 + }, + { + "KL/chosen_KL_mean": -25.55727195739746, + "KL/mean": -44.37687683105469, + "KL/rejected_KL_mean": -63.19647979736328, + "KL/std": 42.76707458496094, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.012777217663824558, + "fcm_dpo/delta": -0.10056991130113602, + "fcm_dpo/margin": 37.63921356201172, + "fcm_dpo/q_t": 0.3908533453941345, + "grad_norm": 19.247800827026367, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.6914454102516174, + "logits/rejected": -0.6689245104789734, + "logps/chosen": -87.71424865722656, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -159.79249572753906, + "loss": 1.0459, + "margin_dpo/margin_mean": 37.63921356201172, + "margin_dpo/margin_std": 47.915870666503906, + "step": 96 + }, + { + "KL/chosen_KL_mean": -26.668346405029297, + "KL/mean": -40.93095779418945, + "KL/rejected_KL_mean": -55.193572998046875, + "KL/std": 36.4090576171875, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.013283725827932358, + "fcm_dpo/delta": 0.019337870180606842, + "fcm_dpo/margin": 28.525230407714844, + "fcm_dpo/q_t": 0.4114477038383484, + "grad_norm": 20.693225860595703, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6356480121612549, + "logits/rejected": -0.6001313924789429, + "logps/chosen": -81.31471252441406, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -132.1583251953125, + "loss": 1.1047, + "margin_dpo/margin_mean": 28.525230407714844, + "margin_dpo/margin_std": 36.311241149902344, + "step": 97 + }, + { + "KL/chosen_KL_mean": -33.744266510009766, + "KL/mean": -50.10036087036133, + "KL/rejected_KL_mean": -66.45645904541016, + "KL/std": 42.09437561035156, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.012812742963433266, + "fcm_dpo/delta": -0.02222163788974285, + "fcm_dpo/margin": 32.71220016479492, + "fcm_dpo/q_t": 0.40252992510795593, + "grad_norm": 21.795242309570312, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6609380841255188, + "logits/rejected": -0.6497888565063477, + "logps/chosen": -99.00289916992188, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -162.98394775390625, + "loss": 1.112, + "margin_dpo/margin_mean": 32.71220016479492, + "margin_dpo/margin_std": 51.15899658203125, + "step": 98 + }, + { + "KL/chosen_KL_mean": -25.77727508544922, + "KL/mean": -46.542884826660156, + "KL/rejected_KL_mean": -67.3084945678711, + "KL/std": 44.52217102050781, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.01204298809170723, + "fcm_dpo/delta": -0.12012702971696854, + "fcm_dpo/margin": 41.531219482421875, + "fcm_dpo/q_t": 0.3889574408531189, + "grad_norm": 18.323759078979492, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6260590553283691, + "logits/rejected": -0.6303242444992065, + "logps/chosen": -71.41575622558594, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -153.74642944335938, + "loss": 1.0435, + "margin_dpo/margin_mean": 41.531219482421875, + "margin_dpo/margin_std": 56.01457214355469, + "step": 99 + }, + { + "KL/chosen_KL_mean": -29.431859970092773, + "KL/mean": -40.424476623535156, + "KL/rejected_KL_mean": -51.41709899902344, + "KL/std": 36.281192779541016, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.012050997465848923, + "fcm_dpo/delta": 0.02939269132912159, + "fcm_dpo/margin": 21.985244750976562, + "fcm_dpo/q_t": 0.4390469491481781, + "grad_norm": 20.13775634765625, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6553602814674377, + "logits/rejected": -0.6196008920669556, + "logps/chosen": -87.02584075927734, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -125.47731018066406, + "loss": 1.2178, + "margin_dpo/margin_mean": 21.985244750976562, + "margin_dpo/margin_std": 46.7362060546875, + "step": 100 + }, + { + "KL/chosen_KL_mean": -36.10777282714844, + "KL/mean": -50.07099533081055, + "KL/rejected_KL_mean": -64.03421020507812, + "KL/std": 45.143096923828125, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.012317821383476257, + "fcm_dpo/delta": 0.06186376139521599, + "fcm_dpo/margin": 27.92643928527832, + "fcm_dpo/q_t": 0.4261440634727478, + "grad_norm": 21.051921844482422, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6446216702461243, + "logits/rejected": -0.619130551815033, + "logps/chosen": -97.75662231445312, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -147.22390747070312, + "loss": 1.176, + "margin_dpo/margin_mean": 27.926443099975586, + "margin_dpo/margin_std": 53.96143341064453, + "step": 101 + }, + { + "KL/chosen_KL_mean": -38.21070098876953, + "KL/mean": -48.19486999511719, + "KL/rejected_KL_mean": -58.17904281616211, + "KL/std": 41.877296447753906, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.012640266679227352, + "fcm_dpo/delta": 0.0061372690834105015, + "fcm_dpo/margin": 19.968339920043945, + "fcm_dpo/q_t": 0.442821204662323, + "grad_norm": 24.04066276550293, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.6613872051239014, + "logits/rejected": -0.6226038336753845, + "logps/chosen": -102.28958129882812, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -126.36611938476562, + "loss": 1.2494, + "margin_dpo/margin_mean": 19.968339920043945, + "margin_dpo/margin_std": 50.369728088378906, + "step": 102 + }, + { + "KL/chosen_KL_mean": -31.718181610107422, + "KL/mean": -52.71338653564453, + "KL/rejected_KL_mean": -73.70858764648438, + "KL/std": 47.02476501464844, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.01192430965602398, + "fcm_dpo/delta": -0.11986613273620605, + "fcm_dpo/margin": 41.99040222167969, + "fcm_dpo/q_t": 0.38860392570495605, + "grad_norm": 20.237232208251953, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6318525075912476, + "logits/rejected": -0.6004771590232849, + "logps/chosen": -93.0174560546875, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -167.28131103515625, + "loss": 1.0491, + "margin_dpo/margin_mean": 41.99040222167969, + "margin_dpo/margin_std": 58.265838623046875, + "step": 103 + }, + { + "KL/chosen_KL_mean": -33.12291717529297, + "KL/mean": -53.138893127441406, + "KL/rejected_KL_mean": -73.15486907958984, + "KL/std": 50.859031677246094, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.011464253067970276, + "fcm_dpo/delta": -0.0682276263833046, + "fcm_dpo/margin": 40.031951904296875, + "fcm_dpo/q_t": 0.40155109763145447, + "grad_norm": 19.202058792114258, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6529978513717651, + "logits/rejected": -0.6287548542022705, + "logps/chosen": -87.49568939208984, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -162.71958923339844, + "loss": 1.0936, + "margin_dpo/margin_mean": 40.031951904296875, + "margin_dpo/margin_std": 63.054229736328125, + "step": 104 + }, + { + "KL/chosen_KL_mean": -31.410409927368164, + "KL/mean": -59.821876525878906, + "KL/rejected_KL_mean": -88.23335266113281, + "KL/std": 48.251129150390625, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.01026911847293377, + "fcm_dpo/delta": -0.22254642844200134, + "fcm_dpo/margin": 56.82294464111328, + "fcm_dpo/q_t": 0.3600938618183136, + "grad_norm": 19.213008880615234, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.6173114776611328, + "logits/rejected": -0.5980815887451172, + "logps/chosen": -86.04935455322266, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -186.20687866210938, + "loss": 0.9553, + "margin_dpo/margin_mean": 56.82294464111328, + "margin_dpo/margin_std": 54.21702575683594, + "step": 105 + }, + { + "KL/chosen_KL_mean": -35.53564453125, + "KL/mean": -53.285430908203125, + "KL/rejected_KL_mean": -71.03523254394531, + "KL/std": 43.38680648803711, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.010062070563435555, + "fcm_dpo/delta": 0.04800419509410858, + "fcm_dpo/margin": 35.49958038330078, + "fcm_dpo/q_t": 0.41749516129493713, + "grad_norm": 18.00835609436035, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.5826778411865234, + "logits/rejected": -0.5607549548149109, + "logps/chosen": -90.36853790283203, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -156.25985717773438, + "loss": 1.1158, + "margin_dpo/margin_mean": 35.49958038330078, + "margin_dpo/margin_std": 47.14399719238281, + "step": 106 + }, + { + "KL/chosen_KL_mean": -42.04072570800781, + "KL/mean": -61.96390151977539, + "KL/rejected_KL_mean": -81.88707733154297, + "KL/std": 53.143890380859375, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.010064037516713142, + "fcm_dpo/delta": -0.0020897462964057922, + "fcm_dpo/margin": 39.84634780883789, + "fcm_dpo/q_t": 0.4079068899154663, + "grad_norm": 18.320009231567383, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.639863908290863, + "logits/rejected": -0.6006349325180054, + "logps/chosen": -111.74853515625, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -176.6265869140625, + "loss": 1.1028, + "margin_dpo/margin_mean": 39.84634780883789, + "margin_dpo/margin_std": 56.551353454589844, + "step": 107 + }, + { + "KL/chosen_KL_mean": -36.460853576660156, + "KL/mean": -61.619041442871094, + "KL/rejected_KL_mean": -86.77722930908203, + "KL/std": 59.005958557128906, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.009726589545607567, + "fcm_dpo/delta": -0.10472159087657928, + "fcm_dpo/margin": 50.316375732421875, + "fcm_dpo/q_t": 0.3944263458251953, + "grad_norm": 17.88787841796875, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.6267660856246948, + "logits/rejected": -0.6133134365081787, + "logps/chosen": -92.47074127197266, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -182.5732421875, + "loss": 1.0772, + "margin_dpo/margin_mean": 50.316375732421875, + "margin_dpo/margin_std": 76.515380859375, + "step": 108 + }, + { + "KL/chosen_KL_mean": -34.545753479003906, + "KL/mean": -60.495208740234375, + "KL/rejected_KL_mean": -86.44466400146484, + "KL/std": 55.74847412109375, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.009096445515751839, + "fcm_dpo/delta": -0.0859290212392807, + "fcm_dpo/margin": 51.8989143371582, + "fcm_dpo/q_t": 0.39097392559051514, + "grad_norm": 18.553064346313477, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5725699663162231, + "logits/rejected": -0.5533767938613892, + "logps/chosen": -97.43124389648438, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -185.13040161132812, + "loss": 1.0589, + "margin_dpo/margin_mean": 51.8989143371582, + "margin_dpo/margin_std": 68.94662475585938, + "step": 109 + }, + { + "KL/chosen_KL_mean": -36.13302993774414, + "KL/mean": -59.498931884765625, + "KL/rejected_KL_mean": -82.8648452758789, + "KL/std": 55.806644439697266, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.008456967771053314, + "fcm_dpo/delta": -0.11751759797334671, + "fcm_dpo/margin": 46.7318115234375, + "fcm_dpo/q_t": 0.4073970913887024, + "grad_norm": 15.185154914855957, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.5394201874732971, + "logits/rejected": -0.5061296224594116, + "logps/chosen": -94.88671875, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -162.6148681640625, + "loss": 1.1137, + "margin_dpo/margin_mean": 46.731807708740234, + "margin_dpo/margin_std": 67.97378540039062, + "step": 110 + }, + { + "KL/chosen_KL_mean": -40.66045379638672, + "KL/mean": -67.77729797363281, + "KL/rejected_KL_mean": -94.89413452148438, + "KL/std": 63.49658966064453, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.00815613754093647, + "fcm_dpo/delta": -0.05090083181858063, + "fcm_dpo/margin": 54.233680725097656, + "fcm_dpo/q_t": 0.3993881940841675, + "grad_norm": 16.779705047607422, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.6132886409759521, + "logits/rejected": -0.593494176864624, + "logps/chosen": -109.28456115722656, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -193.322998046875, + "loss": 1.0932, + "margin_dpo/margin_mean": 54.233680725097656, + "margin_dpo/margin_std": 80.48873138427734, + "step": 111 + }, + { + "KL/chosen_KL_mean": -37.53116989135742, + "KL/mean": -54.17732238769531, + "KL/rejected_KL_mean": -70.82347869873047, + "KL/std": 42.614315032958984, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.00834115780889988, + "fcm_dpo/delta": 0.027919700369238853, + "fcm_dpo/margin": 33.29230499267578, + "fcm_dpo/q_t": 0.4350922703742981, + "grad_norm": 16.026575088500977, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.5812044739723206, + "logits/rejected": -0.5499833822250366, + "logps/chosen": -87.78080749511719, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -135.597900390625, + "loss": 1.176, + "margin_dpo/margin_mean": 33.29230499267578, + "margin_dpo/margin_std": 52.93878936767578, + "step": 112 + }, + { + "KL/chosen_KL_mean": -48.90500259399414, + "KL/mean": -66.02964782714844, + "KL/rejected_KL_mean": -83.15428924560547, + "KL/std": 47.26560592651367, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.008774908259510994, + "fcm_dpo/delta": 0.10903479158878326, + "fcm_dpo/margin": 34.24928283691406, + "fcm_dpo/q_t": 0.43178486824035645, + "grad_norm": 17.91684913635254, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.5346338748931885, + "logits/rejected": -0.4947078824043274, + "logps/chosen": -115.61795043945312, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -161.12298583984375, + "loss": 1.158, + "margin_dpo/margin_mean": 34.24928283691406, + "margin_dpo/margin_std": 52.268184661865234, + "step": 113 + }, + { + "KL/chosen_KL_mean": -44.28437805175781, + "KL/mean": -72.6781234741211, + "KL/rejected_KL_mean": -101.07186889648438, + "KL/std": 68.00740051269531, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.008395552635192871, + "fcm_dpo/delta": -0.09355790913105011, + "fcm_dpo/margin": 56.78749084472656, + "fcm_dpo/q_t": 0.39641422033309937, + "grad_norm": 19.671342849731445, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.5491304397583008, + "logits/rejected": -0.5225331783294678, + "logps/chosen": -102.0694580078125, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -188.1815185546875, + "loss": 1.0664, + "margin_dpo/margin_mean": 56.78749084472656, + "margin_dpo/margin_std": 83.15078735351562, + "step": 114 + }, + { + "KL/chosen_KL_mean": -56.604469299316406, + "KL/mean": -75.81980895996094, + "KL/rejected_KL_mean": -95.03514862060547, + "KL/std": 65.74048614501953, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.008766735903918743, + "fcm_dpo/delta": 0.06877291947603226, + "fcm_dpo/margin": 38.430686950683594, + "fcm_dpo/q_t": 0.4290325343608856, + "grad_norm": 24.451086044311523, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.5631723999977112, + "logits/rejected": -0.5508309602737427, + "logps/chosen": -122.1871109008789, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -193.60067749023438, + "loss": 1.2103, + "margin_dpo/margin_mean": 38.43069076538086, + "margin_dpo/margin_std": 88.03782653808594, + "step": 115 + }, + { + "KL/chosen_KL_mean": -41.52287292480469, + "KL/mean": -63.56541442871094, + "KL/rejected_KL_mean": -85.60796356201172, + "KL/std": 54.60905456542969, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.008929261937737465, + "fcm_dpo/delta": 0.005508579313755035, + "fcm_dpo/margin": 44.0850715637207, + "fcm_dpo/q_t": 0.4103013277053833, + "grad_norm": 18.25627899169922, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.565000057220459, + "logits/rejected": -0.562967836856842, + "logps/chosen": -92.92318725585938, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -166.12982177734375, + "loss": 1.0996, + "margin_dpo/margin_mean": 44.0850715637207, + "margin_dpo/margin_std": 60.139198303222656, + "step": 116 + }, + { + "KL/chosen_KL_mean": -54.4649658203125, + "KL/mean": -71.32565307617188, + "KL/rejected_KL_mean": -88.18632507324219, + "KL/std": 57.524383544921875, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.009208546951413155, + "fcm_dpo/delta": 0.09851875901222229, + "fcm_dpo/margin": 33.721351623535156, + "fcm_dpo/q_t": 0.43140047788619995, + "grad_norm": 27.75358772277832, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.5220273733139038, + "logits/rejected": -0.48897624015808105, + "logps/chosen": -123.76337432861328, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -154.7703094482422, + "loss": 1.1872, + "margin_dpo/margin_mean": 33.721351623535156, + "margin_dpo/margin_std": 66.84080505371094, + "step": 117 + }, + { + "KL/chosen_KL_mean": -41.48072814941406, + "KL/mean": -62.975624084472656, + "KL/rejected_KL_mean": -84.47052764892578, + "KL/std": 51.98291015625, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.009359470568597317, + "fcm_dpo/delta": -0.0027602193877100945, + "fcm_dpo/margin": 42.98979568481445, + "fcm_dpo/q_t": 0.4092516303062439, + "grad_norm": 18.643281936645508, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.5751606822013855, + "logits/rejected": -0.5594580769538879, + "logps/chosen": -97.1217041015625, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -160.13958740234375, + "loss": 1.0966, + "margin_dpo/margin_mean": 42.98979568481445, + "margin_dpo/margin_std": 60.324066162109375, + "step": 118 + }, + { + "KL/chosen_KL_mean": -57.252830505371094, + "KL/mean": -78.42536926269531, + "KL/rejected_KL_mean": -99.597900390625, + "KL/std": 58.55084991455078, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.009168696589767933, + "fcm_dpo/delta": 0.010236188769340515, + "fcm_dpo/margin": 42.3450813293457, + "fcm_dpo/q_t": 0.41542544960975647, + "grad_norm": 20.362258911132812, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.5720139741897583, + "logits/rejected": -0.5641052722930908, + "logps/chosen": -130.76303100585938, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -202.5751953125, + "loss": 1.1412, + "margin_dpo/margin_mean": 42.3450813293457, + "margin_dpo/margin_std": 72.98075866699219, + "step": 119 + }, + { + "KL/chosen_KL_mean": -57.04957962036133, + "KL/mean": -86.55354309082031, + "KL/rejected_KL_mean": -116.0574951171875, + "KL/std": 71.63401794433594, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.008779691532254219, + "fcm_dpo/delta": -0.14065004885196686, + "fcm_dpo/margin": 59.007896423339844, + "fcm_dpo/q_t": 0.3822689950466156, + "grad_norm": 21.11440086364746, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.5682621002197266, + "logits/rejected": -0.541315495967865, + "logps/chosen": -133.83041381835938, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -224.08123779296875, + "loss": 1.0327, + "margin_dpo/margin_mean": 59.007896423339844, + "margin_dpo/margin_std": 76.28536987304688, + "step": 120 + }, + { + "KL/chosen_KL_mean": -54.28160858154297, + "KL/mean": -86.91211700439453, + "KL/rejected_KL_mean": -119.54263305664062, + "KL/std": 69.17658996582031, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.007959427312016487, + "fcm_dpo/delta": -0.14574769139289856, + "fcm_dpo/margin": 65.26101684570312, + "fcm_dpo/q_t": 0.38162851333618164, + "grad_norm": 20.98877716064453, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5243557691574097, + "logits/rejected": -0.5225794315338135, + "logps/chosen": -116.07150268554688, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -229.53720092773438, + "loss": 1.0384, + "margin_dpo/margin_mean": 65.26101684570312, + "margin_dpo/margin_std": 86.40126037597656, + "step": 121 + }, + { + "KL/chosen_KL_mean": -48.655616760253906, + "KL/mean": -88.73162841796875, + "KL/rejected_KL_mean": -128.80764770507812, + "KL/std": 77.35166931152344, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.007193242199718952, + "fcm_dpo/delta": -0.21370908617973328, + "fcm_dpo/margin": 80.15203094482422, + "fcm_dpo/q_t": 0.36466366052627563, + "grad_norm": 19.72478675842285, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.4986063838005066, + "logits/rejected": -0.5158952474594116, + "logps/chosen": -95.55783081054688, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -235.5218505859375, + "loss": 0.9688, + "margin_dpo/margin_mean": 80.15203094482422, + "margin_dpo/margin_std": 82.22235870361328, + "step": 122 + }, + { + "KL/chosen_KL_mean": -54.839569091796875, + "KL/mean": -80.63801574707031, + "KL/rejected_KL_mean": -106.43647766113281, + "KL/std": 66.09374237060547, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.006983310449868441, + "fcm_dpo/delta": 0.043627917766571045, + "fcm_dpo/margin": 51.59690475463867, + "fcm_dpo/q_t": 0.420354425907135, + "grad_norm": 18.404876708984375, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.5001442432403564, + "logits/rejected": -0.48770254850387573, + "logps/chosen": -116.17820739746094, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -194.2118682861328, + "loss": 1.1521, + "margin_dpo/margin_mean": 51.59690475463867, + "margin_dpo/margin_std": 89.58872985839844, + "step": 123 + }, + { + "KL/chosen_KL_mean": -63.81550598144531, + "KL/mean": -103.01524353027344, + "KL/rejected_KL_mean": -142.21498107910156, + "KL/std": 92.14735412597656, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.006636175327003002, + "fcm_dpo/delta": -0.1444467306137085, + "fcm_dpo/margin": 78.39946746826172, + "fcm_dpo/q_t": 0.3870224058628082, + "grad_norm": 17.50848960876465, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.4794590473175049, + "logits/rejected": -0.4842977523803711, + "logps/chosen": -135.2638397216797, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -259.7955322265625, + "loss": 1.0628, + "margin_dpo/margin_mean": 78.39947509765625, + "margin_dpo/margin_std": 115.86351013183594, + "step": 124 + }, + { + "KL/chosen_KL_mean": -52.27369689941406, + "KL/mean": -85.86976623535156, + "KL/rejected_KL_mean": -119.46583557128906, + "KL/std": 77.98124694824219, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.006456486880779266, + "fcm_dpo/delta": -0.038845356553792953, + "fcm_dpo/margin": 67.192138671875, + "fcm_dpo/q_t": 0.4029562771320343, + "grad_norm": 16.04224395751953, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.43761587142944336, + "logits/rejected": -0.44332560896873474, + "logps/chosen": -102.41063690185547, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -203.45445251464844, + "loss": 1.0918, + "margin_dpo/margin_mean": 67.192138671875, + "margin_dpo/margin_std": 98.45494079589844, + "step": 125 + }, + { + "KL/chosen_KL_mean": -56.8275146484375, + "KL/mean": -89.02064514160156, + "KL/rejected_KL_mean": -121.21378326416016, + "KL/std": 71.73283386230469, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.006333824247121811, + "fcm_dpo/delta": -0.009114507585763931, + "fcm_dpo/margin": 64.38626861572266, + "fcm_dpo/q_t": 0.40852469205856323, + "grad_norm": 17.67743492126465, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.4789317846298218, + "logits/rejected": -0.4796581268310547, + "logps/chosen": -112.49458312988281, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -219.34353637695312, + "loss": 1.0978, + "margin_dpo/margin_mean": 64.38626861572266, + "margin_dpo/margin_std": 92.2175064086914, + "step": 126 + }, + { + "KL/chosen_KL_mean": -56.04045104980469, + "KL/mean": -78.40089416503906, + "KL/rejected_KL_mean": -100.76133728027344, + "KL/std": 63.104034423828125, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.006692334543913603, + "fcm_dpo/delta": 0.11030134558677673, + "fcm_dpo/margin": 44.720890045166016, + "fcm_dpo/q_t": 0.43136173486709595, + "grad_norm": 18.180307388305664, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.4499157667160034, + "logits/rejected": -0.43410080671310425, + "logps/chosen": -112.59513092041016, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -177.55712890625, + "loss": 1.1771, + "margin_dpo/margin_mean": 44.72089385986328, + "margin_dpo/margin_std": 79.86116790771484, + "step": 127 + }, + { + "KL/chosen_KL_mean": -58.57484436035156, + "KL/mean": -82.85896301269531, + "KL/rejected_KL_mean": -107.14308166503906, + "KL/std": 73.12893676757812, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.007076849229633808, + "fcm_dpo/delta": 0.060365404933691025, + "fcm_dpo/margin": 48.5682373046875, + "fcm_dpo/q_t": 0.42341214418411255, + "grad_norm": 22.50141143798828, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.4698704481124878, + "logits/rejected": -0.4584053158760071, + "logps/chosen": -116.69580078125, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -183.58203125, + "loss": 1.1739, + "margin_dpo/margin_mean": 48.5682373046875, + "margin_dpo/margin_std": 92.75914764404297, + "step": 128 + }, + { + "KL/chosen_KL_mean": -71.50048828125, + "KL/mean": -96.9925537109375, + "KL/rejected_KL_mean": -122.484619140625, + "KL/std": 83.43275451660156, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.007178121246397495, + "fcm_dpo/delta": 0.03808710351586342, + "fcm_dpo/margin": 50.98412322998047, + "fcm_dpo/q_t": 0.4246157705783844, + "grad_norm": 20.14983558654785, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.46669113636016846, + "logits/rejected": -0.459794282913208, + "logps/chosen": -138.4168701171875, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -219.1268310546875, + "loss": 1.1868, + "margin_dpo/margin_mean": 50.98412322998047, + "margin_dpo/margin_std": 110.04112243652344, + "step": 129 + }, + { + "KL/chosen_KL_mean": -51.86681365966797, + "KL/mean": -90.43751525878906, + "KL/rejected_KL_mean": -129.00820922851562, + "KL/std": 80.59832763671875, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.006917397491633892, + "fcm_dpo/delta": -0.16029784083366394, + "fcm_dpo/margin": 77.14140319824219, + "fcm_dpo/q_t": 0.3812750279903412, + "grad_norm": 18.355745315551758, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.40895193815231323, + "logits/rejected": -0.40286046266555786, + "logps/chosen": -96.53366088867188, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -211.78985595703125, + "loss": 1.0394, + "margin_dpo/margin_mean": 77.14139556884766, + "margin_dpo/margin_std": 100.32928466796875, + "step": 130 + }, + { + "KL/chosen_KL_mean": -48.170814514160156, + "KL/mean": -87.71832275390625, + "KL/rejected_KL_mean": -127.26582336425781, + "KL/std": 84.27751922607422, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.006208081264048815, + "fcm_dpo/delta": -0.10756178945302963, + "fcm_dpo/margin": 79.09501647949219, + "fcm_dpo/q_t": 0.3844504952430725, + "grad_norm": 24.27446937561035, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.3953825533390045, + "logits/rejected": -0.40092289447784424, + "logps/chosen": -93.09540557861328, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -215.7098388671875, + "loss": 1.0294, + "margin_dpo/margin_mean": 79.09501647949219, + "margin_dpo/margin_std": 91.44660186767578, + "step": 131 + }, + { + "KL/chosen_KL_mean": -62.53089904785156, + "KL/mean": -94.03605651855469, + "KL/rejected_KL_mean": -125.54121398925781, + "KL/std": 81.84492492675781, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.006138365715742111, + "fcm_dpo/delta": 0.01479797437787056, + "fcm_dpo/margin": 63.01029968261719, + "fcm_dpo/q_t": 0.416471004486084, + "grad_norm": 17.533231735229492, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.45857325196266174, + "logits/rejected": -0.46344107389450073, + "logps/chosen": -121.531982421875, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -213.43336486816406, + "loss": 1.1322, + "margin_dpo/margin_mean": 63.01030349731445, + "margin_dpo/margin_std": 107.01394653320312, + "step": 132 + }, + { + "KL/chosen_KL_mean": -73.70401000976562, + "KL/mean": -101.21989440917969, + "KL/rejected_KL_mean": -128.7357635498047, + "KL/std": 73.13414001464844, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.006320960819721222, + "fcm_dpo/delta": 0.058340031653642654, + "fcm_dpo/margin": 55.03174591064453, + "fcm_dpo/q_t": 0.4236518144607544, + "grad_norm": 23.5048770904541, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.43090447783470154, + "logits/rejected": -0.41882967948913574, + "logps/chosen": -140.30850219726562, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -225.06932067871094, + "loss": 1.1497, + "margin_dpo/margin_mean": 55.03174591064453, + "margin_dpo/margin_std": 94.34498596191406, + "step": 133 + }, + { + "KL/chosen_KL_mean": -57.201560974121094, + "KL/mean": -86.7327651977539, + "KL/rejected_KL_mean": -116.26396179199219, + "KL/std": 69.65283203125, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.006522106938064098, + "fcm_dpo/delta": 0.015777243301272392, + "fcm_dpo/margin": 59.062408447265625, + "fcm_dpo/q_t": 0.4131925404071808, + "grad_norm": 16.96872329711914, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.4189786911010742, + "logits/rejected": -0.41040804982185364, + "logps/chosen": -109.27081298828125, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -203.91848754882812, + "loss": 1.1027, + "margin_dpo/margin_mean": 59.062408447265625, + "margin_dpo/margin_std": 82.31409454345703, + "step": 134 + }, + { + "KL/chosen_KL_mean": -62.467185974121094, + "KL/mean": -106.80340576171875, + "KL/rejected_KL_mean": -151.13963317871094, + "KL/std": 95.00294494628906, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.006014559417963028, + "fcm_dpo/delta": -0.15938910841941833, + "fcm_dpo/margin": 88.67243957519531, + "fcm_dpo/q_t": 0.3810012936592102, + "grad_norm": 19.60321617126465, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.42210158705711365, + "logits/rejected": -0.45571398735046387, + "logps/chosen": -112.821044921875, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -267.119384765625, + "loss": 1.0306, + "margin_dpo/margin_mean": 88.67243957519531, + "margin_dpo/margin_std": 116.26024627685547, + "step": 135 + }, + { + "KL/chosen_KL_mean": -74.17800903320312, + "KL/mean": -103.6968994140625, + "KL/rejected_KL_mean": -133.2157745361328, + "KL/std": 88.34904479980469, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.005938498303294182, + "fcm_dpo/delta": 0.05452149361371994, + "fcm_dpo/margin": 59.037757873535156, + "fcm_dpo/q_t": 0.428546667098999, + "grad_norm": 17.64590835571289, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.44360649585723877, + "logits/rejected": -0.4397875666618347, + "logps/chosen": -139.25051879882812, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -229.53701782226562, + "loss": 1.1702, + "margin_dpo/margin_mean": 59.037757873535156, + "margin_dpo/margin_std": 115.24250793457031, + "step": 136 + }, + { + "KL/chosen_KL_mean": -68.81549072265625, + "KL/mean": -114.8359146118164, + "KL/rejected_KL_mean": -160.85635375976562, + "KL/std": 113.4111099243164, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.005673682317137718, + "fcm_dpo/delta": -0.14579302072525024, + "fcm_dpo/margin": 92.04085540771484, + "fcm_dpo/q_t": 0.3884824514389038, + "grad_norm": 15.848882675170898, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.3792418837547302, + "logits/rejected": -0.4050024151802063, + "logps/chosen": -117.57460021972656, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -274.7201232910156, + "loss": 1.0599, + "margin_dpo/margin_mean": 92.04085540771484, + "margin_dpo/margin_std": 137.79296875, + "step": 137 + }, + { + "KL/chosen_KL_mean": -73.22099304199219, + "KL/mean": -107.53677368164062, + "KL/rejected_KL_mean": -141.85256958007812, + "KL/std": 85.83137512207031, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.0056349434889853, + "fcm_dpo/delta": 0.014691882766783237, + "fcm_dpo/margin": 68.63157653808594, + "fcm_dpo/q_t": 0.4105899930000305, + "grad_norm": 17.747941970825195, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.39606553316116333, + "logits/rejected": -0.38699305057525635, + "logps/chosen": -133.74063110351562, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -235.0495147705078, + "loss": 1.0938, + "margin_dpo/margin_mean": 68.63157653808594, + "margin_dpo/margin_std": 87.69758605957031, + "step": 138 + }, + { + "KL/chosen_KL_mean": -61.787681579589844, + "KL/mean": -101.80259704589844, + "KL/rejected_KL_mean": -141.8175048828125, + "KL/std": 80.42506408691406, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.005530361086130142, + "fcm_dpo/delta": -0.049082156270742416, + "fcm_dpo/margin": 80.02982330322266, + "fcm_dpo/q_t": 0.39961743354797363, + "grad_norm": 15.96881103515625, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.3929751515388489, + "logits/rejected": -0.38671720027923584, + "logps/chosen": -108.67906188964844, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -221.54547119140625, + "loss": 1.0691, + "margin_dpo/margin_mean": 80.02982330322266, + "margin_dpo/margin_std": 106.43284606933594, + "step": 139 + }, + { + "KL/chosen_KL_mean": -74.34764099121094, + "KL/mean": -110.00637817382812, + "KL/rejected_KL_mean": -145.66513061523438, + "KL/std": 88.292236328125, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.005478830076754093, + "fcm_dpo/delta": 0.010492755100131035, + "fcm_dpo/margin": 71.31747436523438, + "fcm_dpo/q_t": 0.4115165174007416, + "grad_norm": 18.478853225708008, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.38018178939819336, + "logits/rejected": -0.36942821741104126, + "logps/chosen": -133.32235717773438, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -228.94923400878906, + "loss": 1.1134, + "margin_dpo/margin_mean": 71.31747436523438, + "margin_dpo/margin_std": 107.75035095214844, + "step": 140 + }, + { + "KL/chosen_KL_mean": -84.36244201660156, + "KL/mean": -118.40705871582031, + "KL/rejected_KL_mean": -152.45164489746094, + "KL/std": 101.11014556884766, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.005562029778957367, + "fcm_dpo/delta": 0.02388141117990017, + "fcm_dpo/margin": 68.08919525146484, + "fcm_dpo/q_t": 0.41459134221076965, + "grad_norm": 22.268421173095703, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.36744964122772217, + "logits/rejected": -0.3581271171569824, + "logps/chosen": -159.4381103515625, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -250.64390563964844, + "loss": 1.1415, + "margin_dpo/margin_mean": 68.08919525146484, + "margin_dpo/margin_std": 117.78225708007812, + "step": 141 + }, + { + "KL/chosen_KL_mean": -83.84671020507812, + "KL/mean": -122.79887390136719, + "KL/rejected_KL_mean": -161.75106811523438, + "KL/std": 106.88426971435547, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.005544648971408606, + "fcm_dpo/delta": -0.0373292937874794, + "fcm_dpo/margin": 77.90434265136719, + "fcm_dpo/q_t": 0.40400242805480957, + "grad_norm": 24.56593894958496, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.3489595055580139, + "logits/rejected": -0.3497436046600342, + "logps/chosen": -141.8746337890625, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -256.3332824707031, + "loss": 1.1167, + "margin_dpo/margin_mean": 77.90434265136719, + "margin_dpo/margin_std": 125.61709594726562, + "step": 142 + }, + { + "KL/chosen_KL_mean": -87.21573638916016, + "KL/mean": -112.6225814819336, + "KL/rejected_KL_mean": -138.0294189453125, + "KL/std": 93.06007385253906, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.0055214231833815575, + "fcm_dpo/delta": 0.023718703538179398, + "fcm_dpo/margin": 50.81369400024414, + "fcm_dpo/q_t": 0.438992977142334, + "grad_norm": 20.991621017456055, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.32056617736816406, + "logits/rejected": -0.3174586296081543, + "logps/chosen": -144.81219482421875, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -217.02899169921875, + "loss": 1.2115, + "margin_dpo/margin_mean": 50.81369400024414, + "margin_dpo/margin_std": 109.45118713378906, + "step": 143 + }, + { + "KL/chosen_KL_mean": -80.57756805419922, + "KL/mean": -110.68243408203125, + "KL/rejected_KL_mean": -140.78729248046875, + "KL/std": 80.64029693603516, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.005678724963217974, + "fcm_dpo/delta": 0.06475970149040222, + "fcm_dpo/margin": 60.209739685058594, + "fcm_dpo/q_t": 0.4229664206504822, + "grad_norm": 20.44149398803711, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.31653550267219543, + "logits/rejected": -0.31111419200897217, + "logps/chosen": -140.4839324951172, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -222.78756713867188, + "loss": 1.1409, + "margin_dpo/margin_mean": 60.209739685058594, + "margin_dpo/margin_std": 94.09750366210938, + "step": 144 + }, + { + "KL/chosen_KL_mean": -78.03926086425781, + "KL/mean": -110.17340087890625, + "KL/rejected_KL_mean": -142.3075408935547, + "KL/std": 79.70943450927734, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.005811285227537155, + "fcm_dpo/delta": 0.02988569624722004, + "fcm_dpo/margin": 64.26829528808594, + "fcm_dpo/q_t": 0.4154420495033264, + "grad_norm": 23.59065055847168, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.3525102138519287, + "logits/rejected": -0.3427172303199768, + "logps/chosen": -134.63992309570312, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -220.17385864257812, + "loss": 1.1234, + "margin_dpo/margin_mean": 64.26829528808594, + "margin_dpo/margin_std": 98.41337585449219, + "step": 145 + }, + { + "KL/chosen_KL_mean": -104.3462905883789, + "KL/mean": -129.30636596679688, + "KL/rejected_KL_mean": -154.26644897460938, + "KL/std": 87.92964172363281, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.006070768926292658, + "fcm_dpo/delta": 0.10701952874660492, + "fcm_dpo/margin": 49.920166015625, + "fcm_dpo/q_t": 0.4325924217700958, + "grad_norm": 33.955230712890625, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.34393632411956787, + "logits/rejected": -0.327251672744751, + "logps/chosen": -170.34674072265625, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -235.96923828125, + "loss": 1.2083, + "margin_dpo/margin_mean": 49.920162200927734, + "margin_dpo/margin_std": 105.13346862792969, + "step": 146 + }, + { + "KL/chosen_KL_mean": -75.93870544433594, + "KL/mean": -109.69927978515625, + "KL/rejected_KL_mean": -143.45986938476562, + "KL/std": 86.81598663330078, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.006153445690870285, + "fcm_dpo/delta": -0.017937855795025826, + "fcm_dpo/margin": 67.52116394042969, + "fcm_dpo/q_t": 0.4079192876815796, + "grad_norm": 20.214391708374023, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.32991477847099304, + "logits/rejected": -0.3135877251625061, + "logps/chosen": -129.3441925048828, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -214.85047912597656, + "loss": 1.1151, + "margin_dpo/margin_mean": 67.52116394042969, + "margin_dpo/margin_std": 108.63333129882812, + "step": 147 + }, + { + "KL/chosen_KL_mean": -75.04109954833984, + "KL/mean": -104.97638702392578, + "KL/rejected_KL_mean": -134.91168212890625, + "KL/std": 88.82324981689453, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.0060166651383042336, + "fcm_dpo/delta": -0.08214031904935837, + "fcm_dpo/margin": 59.870567321777344, + "fcm_dpo/q_t": 0.42001479864120483, + "grad_norm": 18.194866180419922, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.4171934127807617, + "logits/rejected": -0.4128437340259552, + "logps/chosen": -139.97817993164062, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -238.00552368164062, + "loss": 1.1426, + "margin_dpo/margin_mean": 59.870567321777344, + "margin_dpo/margin_std": 94.12532043457031, + "step": 148 + }, + { + "KL/chosen_KL_mean": -70.27203369140625, + "KL/mean": -107.100341796875, + "KL/rejected_KL_mean": -143.92864990234375, + "KL/std": 77.69738006591797, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.005668614059686661, + "fcm_dpo/delta": -0.024732418358325958, + "fcm_dpo/margin": 73.6566162109375, + "fcm_dpo/q_t": 0.40335261821746826, + "grad_norm": 16.857189178466797, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.41874703764915466, + "logits/rejected": -0.40718960762023926, + "logps/chosen": -128.7458038330078, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -243.243408203125, + "loss": 1.0821, + "margin_dpo/margin_mean": 73.6566162109375, + "margin_dpo/margin_std": 92.89242553710938, + "step": 149 + }, + { + "KL/chosen_KL_mean": -61.69485092163086, + "KL/mean": -98.55900573730469, + "KL/rejected_KL_mean": -135.42315673828125, + "KL/std": 91.86660766601562, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.005610861349850893, + "fcm_dpo/delta": -0.019392184913158417, + "fcm_dpo/margin": 73.72830963134766, + "fcm_dpo/q_t": 0.4093967080116272, + "grad_norm": 17.780986785888672, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.4064145088195801, + "logits/rejected": -0.41252338886260986, + "logps/chosen": -107.40066528320312, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -218.770751953125, + "loss": 1.0992, + "margin_dpo/margin_mean": 73.72830200195312, + "margin_dpo/margin_std": 109.19731903076172, + "step": 150 + }, + { + "KL/chosen_KL_mean": -78.20762634277344, + "KL/mean": -114.00714111328125, + "KL/rejected_KL_mean": -149.80667114257812, + "KL/std": 87.08969116210938, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.0056825121864676476, + "fcm_dpo/delta": -0.007870430126786232, + "fcm_dpo/margin": 71.59902954101562, + "fcm_dpo/q_t": 0.40664440393447876, + "grad_norm": 20.973403930664062, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.4537152051925659, + "logits/rejected": -0.42913973331451416, + "logps/chosen": -148.77847290039062, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -250.27047729492188, + "loss": 1.0886, + "margin_dpo/margin_mean": 71.59902954101562, + "margin_dpo/margin_std": 95.69638061523438, + "step": 151 + }, + { + "KL/chosen_KL_mean": -67.67739868164062, + "KL/mean": -111.67888641357422, + "KL/rejected_KL_mean": -155.6803741455078, + "KL/std": 88.04423522949219, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.0054802740924060345, + "fcm_dpo/delta": -0.0959089919924736, + "fcm_dpo/margin": 88.00296783447266, + "fcm_dpo/q_t": 0.39007505774497986, + "grad_norm": 23.02900505065918, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.44229328632354736, + "logits/rejected": -0.44293320178985596, + "logps/chosen": -127.84178924560547, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -261.8208312988281, + "loss": 1.0442, + "margin_dpo/margin_mean": 88.00296783447266, + "margin_dpo/margin_std": 108.72247314453125, + "step": 152 + }, + { + "KL/chosen_KL_mean": -66.60357666015625, + "KL/mean": -103.91030883789062, + "KL/rejected_KL_mean": -141.217041015625, + "KL/std": 95.7032699584961, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.0053931656293570995, + "fcm_dpo/delta": -0.0029367562383413315, + "fcm_dpo/margin": 74.61346435546875, + "fcm_dpo/q_t": 0.4119097590446472, + "grad_norm": 15.23172378540039, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.454341858625412, + "logits/rejected": -0.45268675684928894, + "logps/chosen": -122.91885375976562, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -226.87286376953125, + "loss": 1.1143, + "margin_dpo/margin_mean": 74.61346435546875, + "margin_dpo/margin_std": 118.15116882324219, + "step": 153 + }, + { + "KL/chosen_KL_mean": -79.47211456298828, + "KL/mean": -114.15584564208984, + "KL/rejected_KL_mean": -148.83956909179688, + "KL/std": 93.29608917236328, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.005473325029015541, + "fcm_dpo/delta": 0.021816356107592583, + "fcm_dpo/margin": 69.36743927001953, + "fcm_dpo/q_t": 0.4157505929470062, + "grad_norm": 18.06127166748047, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.43713948130607605, + "logits/rejected": -0.4419820308685303, + "logps/chosen": -142.21469116210938, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -253.08377075195312, + "loss": 1.1461, + "margin_dpo/margin_mean": 69.36744689941406, + "margin_dpo/margin_std": 121.77591705322266, + "step": 154 + }, + { + "KL/chosen_KL_mean": -71.95777893066406, + "KL/mean": -106.21212005615234, + "KL/rejected_KL_mean": -140.46646118164062, + "KL/std": 85.34565734863281, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.005531606264412403, + "fcm_dpo/delta": 0.023080263286828995, + "fcm_dpo/margin": 68.5086898803711, + "fcm_dpo/q_t": 0.4144378900527954, + "grad_norm": 18.350255966186523, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.45376110076904297, + "logits/rejected": -0.43682199716567993, + "logps/chosen": -132.6109619140625, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -217.95867919921875, + "loss": 1.1253, + "margin_dpo/margin_mean": 68.5086898803711, + "margin_dpo/margin_std": 108.21893310546875, + "step": 155 + }, + { + "KL/chosen_KL_mean": -97.99627685546875, + "KL/mean": -117.68576049804688, + "KL/rejected_KL_mean": -137.375244140625, + "KL/std": 90.27848815917969, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.005721476394683123, + "fcm_dpo/delta": 0.06367640942335129, + "fcm_dpo/margin": 39.37895202636719, + "fcm_dpo/q_t": 0.4493403434753418, + "grad_norm": 33.21921920776367, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.4051647484302521, + "logits/rejected": -0.4019678235054016, + "logps/chosen": -167.4881591796875, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -214.54452514648438, + "loss": 1.2843, + "margin_dpo/margin_mean": 39.37895202636719, + "margin_dpo/margin_std": 119.69397735595703, + "step": 156 + }, + { + "KL/chosen_KL_mean": -86.41752624511719, + "KL/mean": -131.41390991210938, + "KL/rejected_KL_mean": -176.4102783203125, + "KL/std": 98.7552490234375, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.005377953872084618, + "fcm_dpo/delta": -0.10211771726608276, + "fcm_dpo/margin": 89.99275207519531, + "fcm_dpo/q_t": 0.38601139187812805, + "grad_norm": 21.639942169189453, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.34451037645339966, + "logits/rejected": -0.3507017493247986, + "logps/chosen": -147.7859649658203, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -284.056640625, + "loss": 1.0489, + "margin_dpo/margin_mean": 89.99275207519531, + "margin_dpo/margin_std": 111.48163604736328, + "step": 157 + }, + { + "KL/chosen_KL_mean": -87.82221984863281, + "KL/mean": -136.22897338867188, + "KL/rejected_KL_mean": -184.6357421875, + "KL/std": 122.65851593017578, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.005120456218719482, + "fcm_dpo/delta": -0.11252043396234512, + "fcm_dpo/margin": 96.81353759765625, + "fcm_dpo/q_t": 0.3937973380088806, + "grad_norm": 16.93302345275879, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.3737766742706299, + "logits/rejected": -0.39256834983825684, + "logps/chosen": -145.4351348876953, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -298.3304138183594, + "loss": 1.0752, + "margin_dpo/margin_mean": 96.81353759765625, + "margin_dpo/margin_std": 149.20346069335938, + "step": 158 + }, + { + "KL/chosen_KL_mean": -102.094970703125, + "KL/mean": -134.10972595214844, + "KL/rejected_KL_mean": -166.12448120117188, + "KL/std": 108.83506774902344, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.005254849791526794, + "fcm_dpo/delta": 0.0684753954410553, + "fcm_dpo/margin": 64.02951049804688, + "fcm_dpo/q_t": 0.4246983528137207, + "grad_norm": 19.485450744628906, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.3579848110675812, + "logits/rejected": -0.32603466510772705, + "logps/chosen": -183.65530395507812, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -255.023193359375, + "loss": 1.171, + "margin_dpo/margin_mean": 64.02951049804688, + "margin_dpo/margin_std": 118.79083251953125, + "step": 159 + }, + { + "KL/chosen_KL_mean": -103.38423156738281, + "KL/mean": -147.4956817626953, + "KL/rejected_KL_mean": -191.60714721679688, + "KL/std": 115.12648010253906, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.0050381142646074295, + "fcm_dpo/delta": -0.05396723747253418, + "fcm_dpo/margin": 88.2229232788086, + "fcm_dpo/q_t": 0.4035095274448395, + "grad_norm": 22.84100341796875, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.3388751745223999, + "logits/rejected": -0.3312157690525055, + "logps/chosen": -169.1151123046875, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -288.8249816894531, + "loss": 1.1098, + "margin_dpo/margin_mean": 88.2229232788086, + "margin_dpo/margin_std": 144.72088623046875, + "step": 160 + }, + { + "KL/chosen_KL_mean": -87.86138916015625, + "KL/mean": -126.76461029052734, + "KL/rejected_KL_mean": -165.66783142089844, + "KL/std": 91.1479263305664, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.0050827739760279655, + "fcm_dpo/delta": 0.005137493368238211, + "fcm_dpo/margin": 77.80645751953125, + "fcm_dpo/q_t": 0.411297082901001, + "grad_norm": 18.5627498626709, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.3419128358364105, + "logits/rejected": -0.342398464679718, + "logps/chosen": -140.2978515625, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -249.09878540039062, + "loss": 1.1085, + "margin_dpo/margin_mean": 77.80645751953125, + "margin_dpo/margin_std": 114.78633117675781, + "step": 161 + }, + { + "KL/chosen_KL_mean": -86.84844970703125, + "KL/mean": -124.30550384521484, + "KL/rejected_KL_mean": -161.7625732421875, + "KL/std": 101.18778991699219, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.0050215162336826324, + "fcm_dpo/delta": 0.023272335529327393, + "fcm_dpo/margin": 74.91412353515625, + "fcm_dpo/q_t": 0.416292667388916, + "grad_norm": 19.82974624633789, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.4207826256752014, + "logits/rejected": -0.4087788462638855, + "logps/chosen": -149.4590301513672, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -251.15313720703125, + "loss": 1.137, + "margin_dpo/margin_mean": 74.91412353515625, + "margin_dpo/margin_std": 121.48768615722656, + "step": 162 + }, + { + "KL/chosen_KL_mean": -98.25493621826172, + "KL/mean": -134.6405029296875, + "KL/rejected_KL_mean": -171.02606201171875, + "KL/std": 102.80780029296875, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.005219772923737764, + "fcm_dpo/delta": 0.022703539580106735, + "fcm_dpo/margin": 72.77113342285156, + "fcm_dpo/q_t": 0.41694432497024536, + "grad_norm": 19.562599182128906, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.3066878914833069, + "logits/rejected": -0.2890280485153198, + "logps/chosen": -153.2765655517578, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -246.4442901611328, + "loss": 1.1378, + "margin_dpo/margin_mean": 72.77113342285156, + "margin_dpo/margin_std": 124.26139831542969, + "step": 163 + }, + { + "KL/chosen_KL_mean": -90.68274688720703, + "KL/mean": -136.00204467773438, + "KL/rejected_KL_mean": -181.32135009765625, + "KL/std": 99.61184692382812, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.0049817366525530815, + "fcm_dpo/delta": -0.06410413980484009, + "fcm_dpo/margin": 90.63861083984375, + "fcm_dpo/q_t": 0.39530429244041443, + "grad_norm": 20.18059730529785, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.32884663343429565, + "logits/rejected": -0.3182298541069031, + "logps/chosen": -146.32342529296875, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -260.9859924316406, + "loss": 1.0617, + "margin_dpo/margin_mean": 90.63861083984375, + "margin_dpo/margin_std": 114.39100646972656, + "step": 164 + }, + { + "KL/chosen_KL_mean": -97.927001953125, + "KL/mean": -127.70655059814453, + "KL/rejected_KL_mean": -157.48609924316406, + "KL/std": 86.75706481933594, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.00522049842402339, + "fcm_dpo/delta": 0.0985773429274559, + "fcm_dpo/margin": 59.55909729003906, + "fcm_dpo/q_t": 0.43164053559303284, + "grad_norm": 22.413503646850586, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.32691213488578796, + "logits/rejected": -0.3019031882286072, + "logps/chosen": -159.23770141601562, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -231.15670776367188, + "loss": 1.1868, + "margin_dpo/margin_mean": 59.5590934753418, + "margin_dpo/margin_std": 115.67192840576172, + "step": 165 + }, + { + "KL/chosen_KL_mean": -87.07829284667969, + "KL/mean": -142.81613159179688, + "KL/rejected_KL_mean": -198.5540008544922, + "KL/std": 125.95869445800781, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.0048718322068452835, + "fcm_dpo/delta": -0.175408735871315, + "fcm_dpo/margin": 111.47569274902344, + "fcm_dpo/q_t": 0.38476717472076416, + "grad_norm": 18.412582397460938, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.2890860438346863, + "logits/rejected": -0.3002024292945862, + "logps/chosen": -138.0618896484375, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -296.64910888671875, + "loss": 1.0389, + "margin_dpo/margin_mean": 111.4756851196289, + "margin_dpo/margin_std": 159.23684692382812, + "step": 166 + }, + { + "KL/chosen_KL_mean": -88.18852233886719, + "KL/mean": -146.77752685546875, + "KL/rejected_KL_mean": -205.3665008544922, + "KL/std": 114.04957580566406, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.004490291699767113, + "fcm_dpo/delta": -0.1502971202135086, + "fcm_dpo/margin": 117.177978515625, + "fcm_dpo/q_t": 0.37806665897369385, + "grad_norm": 17.076223373413086, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.2766282558441162, + "logits/rejected": -0.28032341599464417, + "logps/chosen": -138.61260986328125, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -301.39691162109375, + "loss": 1.0084, + "margin_dpo/margin_mean": 117.177978515625, + "margin_dpo/margin_std": 133.40286254882812, + "step": 167 + }, + { + "KL/chosen_KL_mean": -95.22886657714844, + "KL/mean": -137.9364776611328, + "KL/rejected_KL_mean": -180.64410400390625, + "KL/std": 107.20812225341797, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.004418864846229553, + "fcm_dpo/delta": 0.025371143594384193, + "fcm_dpo/margin": 85.41524505615234, + "fcm_dpo/q_t": 0.41491666436195374, + "grad_norm": 21.82142448425293, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.2899581789970398, + "logits/rejected": -0.28221678733825684, + "logps/chosen": -144.69168090820312, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -255.95266723632812, + "loss": 1.1109, + "margin_dpo/margin_mean": 85.41524505615234, + "margin_dpo/margin_std": 121.71943664550781, + "step": 168 + }, + { + "KL/chosen_KL_mean": -99.43185424804688, + "KL/mean": -135.762451171875, + "KL/rejected_KL_mean": -172.093017578125, + "KL/std": 104.55729675292969, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.004351547919213772, + "fcm_dpo/delta": -0.04128072038292885, + "fcm_dpo/margin": 72.66116333007812, + "fcm_dpo/q_t": 0.42972636222839355, + "grad_norm": 21.818082809448242, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.3337467908859253, + "logits/rejected": -0.3205113112926483, + "logps/chosen": -159.23529052734375, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -255.4387664794922, + "loss": 1.1883, + "margin_dpo/margin_mean": 72.66117095947266, + "margin_dpo/margin_std": 144.61907958984375, + "step": 169 + }, + { + "KL/chosen_KL_mean": -87.06556701660156, + "KL/mean": -130.15606689453125, + "KL/rejected_KL_mean": -173.2465362548828, + "KL/std": 100.10464477539062, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.004333356395363808, + "fcm_dpo/delta": 0.02834583818912506, + "fcm_dpo/margin": 86.18096923828125, + "fcm_dpo/q_t": 0.41411614418029785, + "grad_norm": 16.478069305419922, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.3138449788093567, + "logits/rejected": -0.3056906461715698, + "logps/chosen": -136.53733825683594, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -249.16387939453125, + "loss": 1.1155, + "margin_dpo/margin_mean": 86.18096923828125, + "margin_dpo/margin_std": 120.68376159667969, + "step": 170 + }, + { + "KL/chosen_KL_mean": -126.3691177368164, + "KL/mean": -161.46255493164062, + "KL/rejected_KL_mean": -196.5560302734375, + "KL/std": 117.38258361816406, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.004591408651322126, + "fcm_dpo/delta": 0.08618248999118805, + "fcm_dpo/margin": 70.18690490722656, + "fcm_dpo/q_t": 0.4324600398540497, + "grad_norm": 22.906251907348633, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.316098690032959, + "logits/rejected": -0.3037240207195282, + "logps/chosen": -210.86843872070312, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -305.9381103515625, + "loss": 1.1957, + "margin_dpo/margin_mean": 70.18690490722656, + "margin_dpo/margin_std": 150.94085693359375, + "step": 171 + }, + { + "KL/chosen_KL_mean": -109.90959167480469, + "KL/mean": -146.52381896972656, + "KL/rejected_KL_mean": -183.13803100585938, + "KL/std": 112.02574157714844, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.004770576022565365, + "fcm_dpo/delta": 0.056669969111680984, + "fcm_dpo/margin": 73.22843933105469, + "fcm_dpo/q_t": 0.4222393333911896, + "grad_norm": 18.645238876342773, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.3444163203239441, + "logits/rejected": -0.3300691843032837, + "logps/chosen": -178.56350708007812, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -268.57470703125, + "loss": 1.1509, + "margin_dpo/margin_mean": 73.22843933105469, + "margin_dpo/margin_std": 126.54878234863281, + "step": 172 + }, + { + "KL/chosen_KL_mean": -99.76567840576172, + "KL/mean": -136.50550842285156, + "KL/rejected_KL_mean": -173.24533081054688, + "KL/std": 105.62751770019531, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.004963952116668224, + "fcm_dpo/delta": 0.03807845339179039, + "fcm_dpo/margin": 73.47966003417969, + "fcm_dpo/q_t": 0.41807180643081665, + "grad_norm": 23.83882713317871, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.38224536180496216, + "logits/rejected": -0.3574422597885132, + "logps/chosen": -162.81655883789062, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -251.92926025390625, + "loss": 1.1243, + "margin_dpo/margin_mean": 73.47966003417969, + "margin_dpo/margin_std": 110.53981018066406, + "step": 173 + }, + { + "KL/chosen_KL_mean": -92.5399169921875, + "KL/mean": -136.5845947265625, + "KL/rejected_KL_mean": -180.6292724609375, + "KL/std": 106.97665405273438, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.004934686701744795, + "fcm_dpo/delta": -0.041495803743600845, + "fcm_dpo/margin": 88.08935546875, + "fcm_dpo/q_t": 0.40332791209220886, + "grad_norm": 21.127315521240234, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.2742489278316498, + "logits/rejected": -0.293137788772583, + "logps/chosen": -145.90289306640625, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -282.54046630859375, + "loss": 1.0945, + "margin_dpo/margin_mean": 88.08935546875, + "margin_dpo/margin_std": 129.7069091796875, + "step": 174 + }, + { + "KL/chosen_KL_mean": -84.59925842285156, + "KL/mean": -143.55201721191406, + "KL/rejected_KL_mean": -202.5047607421875, + "KL/std": 113.11393737792969, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.004472202621400356, + "fcm_dpo/delta": -0.152583509683609, + "fcm_dpo/margin": 117.90550231933594, + "fcm_dpo/q_t": 0.3748548626899719, + "grad_norm": 26.67053985595703, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.2666695713996887, + "logits/rejected": -0.2526741921901703, + "logps/chosen": -130.01702880859375, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -292.01055908203125, + "loss": 0.9909, + "margin_dpo/margin_mean": 117.90550994873047, + "margin_dpo/margin_std": 118.56768798828125, + "step": 175 + }, + { + "KL/chosen_KL_mean": -90.1077880859375, + "KL/mean": -141.28628540039062, + "KL/rejected_KL_mean": -192.46481323242188, + "KL/std": 114.48770904541016, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.004313723649829626, + "fcm_dpo/delta": -0.04780307412147522, + "fcm_dpo/margin": 102.35701751708984, + "fcm_dpo/q_t": 0.40044766664505005, + "grad_norm": 17.87701416015625, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.28547635674476624, + "logits/rejected": -0.3026646375656128, + "logps/chosen": -140.56063842773438, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -288.0237731933594, + "loss": 1.0651, + "margin_dpo/margin_mean": 102.35700988769531, + "margin_dpo/margin_std": 131.352783203125, + "step": 176 + }, + { + "KL/chosen_KL_mean": -104.84869384765625, + "KL/mean": -155.22686767578125, + "KL/rejected_KL_mean": -205.60504150390625, + "KL/std": 124.53485107421875, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.004244742915034294, + "fcm_dpo/delta": -0.03202240169048309, + "fcm_dpo/margin": 100.75636291503906, + "fcm_dpo/q_t": 0.4032580554485321, + "grad_norm": 22.515695571899414, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.32991307973861694, + "logits/rejected": -0.3236857056617737, + "logps/chosen": -166.06515502929688, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -301.49884033203125, + "loss": 1.0741, + "margin_dpo/margin_mean": 100.75636291503906, + "margin_dpo/margin_std": 131.8935089111328, + "step": 177 + }, + { + "KL/chosen_KL_mean": -118.5032958984375, + "KL/mean": -180.68392944335938, + "KL/rejected_KL_mean": -242.86456298828125, + "KL/std": 145.39337158203125, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.003944427706301212, + "fcm_dpo/delta": -0.10848015546798706, + "fcm_dpo/margin": 124.36128234863281, + "fcm_dpo/q_t": 0.3889586329460144, + "grad_norm": 26.35658836364746, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.20839257538318634, + "logits/rejected": -0.1991354376077652, + "logps/chosen": -176.76808166503906, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -348.22991943359375, + "loss": 1.0416, + "margin_dpo/margin_mean": 124.36128997802734, + "margin_dpo/margin_std": 159.1615447998047, + "step": 178 + }, + { + "KL/chosen_KL_mean": -124.11444854736328, + "KL/mean": -167.19317626953125, + "KL/rejected_KL_mean": -210.27188110351562, + "KL/std": 127.0611572265625, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.003985902294516563, + "fcm_dpo/delta": 0.0628318265080452, + "fcm_dpo/margin": 86.15742492675781, + "fcm_dpo/q_t": 0.4241906404495239, + "grad_norm": 30.835369110107422, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.26868346333503723, + "logits/rejected": -0.2806195616722107, + "logps/chosen": -185.17276000976562, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -300.7997131347656, + "loss": 1.1691, + "margin_dpo/margin_mean": 86.15742492675781, + "margin_dpo/margin_std": 159.85833740234375, + "step": 179 + }, + { + "KL/chosen_KL_mean": -109.68701171875, + "KL/mean": -166.80178833007812, + "KL/rejected_KL_mean": -223.9165496826172, + "KL/std": 112.77030944824219, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.003952879458665848, + "fcm_dpo/delta": -0.059922002255916595, + "fcm_dpo/margin": 114.22952270507812, + "fcm_dpo/q_t": 0.39513441920280457, + "grad_norm": 18.506088256835938, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.2562364935874939, + "logits/rejected": -0.2543151080608368, + "logps/chosen": -164.0297393798828, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -322.1283874511719, + "loss": 1.0573, + "margin_dpo/margin_mean": 114.22952270507812, + "margin_dpo/margin_std": 142.00148010253906, + "step": 180 + }, + { + "KL/chosen_KL_mean": -94.9785385131836, + "KL/mean": -130.97708129882812, + "KL/rejected_KL_mean": -166.97564697265625, + "KL/std": 106.71527862548828, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.004145805723965168, + "fcm_dpo/delta": 0.11085755378007889, + "fcm_dpo/margin": 71.99710083007812, + "fcm_dpo/q_t": 0.4335516691207886, + "grad_norm": 18.31833267211914, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.26776430010795593, + "logits/rejected": -0.243585467338562, + "logps/chosen": -149.97900390625, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -228.63180541992188, + "loss": 1.185, + "margin_dpo/margin_mean": 71.99708557128906, + "margin_dpo/margin_std": 136.76458740234375, + "step": 181 + }, + { + "KL/chosen_KL_mean": -92.40129089355469, + "KL/mean": -156.1376953125, + "KL/rejected_KL_mean": -219.87408447265625, + "KL/std": 130.09426879882812, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.003931665793061256, + "fcm_dpo/delta": -0.12176034599542618, + "fcm_dpo/margin": 127.47280883789062, + "fcm_dpo/q_t": 0.3830812871456146, + "grad_norm": 18.976547241210938, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.2367890179157257, + "logits/rejected": -0.24935263395309448, + "logps/chosen": -133.50914001464844, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -309.39569091796875, + "loss": 1.0348, + "margin_dpo/margin_mean": 127.47280883789062, + "margin_dpo/margin_std": 157.15118408203125, + "step": 182 + }, + { + "KL/chosen_KL_mean": -134.46290588378906, + "KL/mean": -167.45004272460938, + "KL/rejected_KL_mean": -200.43716430664062, + "KL/std": 107.40684509277344, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.0038527981378138065, + "fcm_dpo/delta": -0.021784789860248566, + "fcm_dpo/margin": 65.97425842285156, + "fcm_dpo/q_t": 0.4400058388710022, + "grad_norm": 19.29417610168457, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.2482393980026245, + "logits/rejected": -0.2410772144794464, + "logps/chosen": -191.9874725341797, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -276.41290283203125, + "loss": 1.2052, + "margin_dpo/margin_mean": 65.97425079345703, + "margin_dpo/margin_std": 117.69155883789062, + "step": 183 + }, + { + "KL/chosen_KL_mean": -114.93549346923828, + "KL/mean": -150.45193481445312, + "KL/rejected_KL_mean": -185.96836853027344, + "KL/std": 102.05284881591797, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.003871684428304434, + "fcm_dpo/delta": 0.016260193660855293, + "fcm_dpo/margin": 71.03287506103516, + "fcm_dpo/q_t": 0.4372020363807678, + "grad_norm": 18.99155616760254, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.29051637649536133, + "logits/rejected": -0.28193700313568115, + "logps/chosen": -173.48043823242188, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -262.6024169921875, + "loss": 1.1912, + "margin_dpo/margin_mean": 71.03286743164062, + "margin_dpo/margin_std": 129.46243286132812, + "step": 184 + }, + { + "KL/chosen_KL_mean": -123.68155670166016, + "KL/mean": -151.2360076904297, + "KL/rejected_KL_mean": -178.79046630859375, + "KL/std": 117.13897705078125, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.0038905704859644175, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 55.10888671875, + "fcm_dpo/q_t": 0.45287707448005676, + "grad_norm": 18.670530319213867, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.23688596487045288, + "logits/rejected": -0.2131071239709854, + "logps/chosen": -185.7073974609375, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -252.5530548095703, + "loss": 1.254, + "margin_dpo/margin_mean": 55.108882904052734, + "margin_dpo/margin_std": 142.17550659179688, + "step": 185 + }, + { + "KL/chosen_KL_mean": -111.7707290649414, + "KL/mean": -164.47586059570312, + "KL/rejected_KL_mean": -217.18099975585938, + "KL/std": 114.5178451538086, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.003862186800688505, + "fcm_dpo/delta": -0.008232121355831623, + "fcm_dpo/margin": 105.41029357910156, + "fcm_dpo/q_t": 0.40653279423713684, + "grad_norm": 21.691234588623047, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.3356194496154785, + "logits/rejected": -0.3203178644180298, + "logps/chosen": -181.12420654296875, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -305.25347900390625, + "loss": 1.0814, + "margin_dpo/margin_mean": 105.41029357910156, + "margin_dpo/margin_std": 133.4476318359375, + "step": 186 + }, + { + "KL/chosen_KL_mean": -101.48190307617188, + "KL/mean": -145.54522705078125, + "KL/rejected_KL_mean": -189.60858154296875, + "KL/std": 108.31391906738281, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.004006706178188324, + "fcm_dpo/delta": 0.050856247544288635, + "fcm_dpo/margin": 88.12667083740234, + "fcm_dpo/q_t": 0.41868123412132263, + "grad_norm": 19.385086059570312, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.28911092877388, + "logits/rejected": -0.2830836772918701, + "logps/chosen": -154.2383575439453, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -271.5776672363281, + "loss": 1.1176, + "margin_dpo/margin_mean": 88.12666320800781, + "margin_dpo/margin_std": 117.51254272460938, + "step": 187 + }, + { + "KL/chosen_KL_mean": -92.61531066894531, + "KL/mean": -145.641357421875, + "KL/rejected_KL_mean": -198.66741943359375, + "KL/std": 118.34048461914062, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.003959735855460167, + "fcm_dpo/delta": -0.02279561012983322, + "fcm_dpo/margin": 106.05211639404297, + "fcm_dpo/q_t": 0.40372931957244873, + "grad_norm": 25.937713623046875, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.2979399561882019, + "logits/rejected": -0.30483749508857727, + "logps/chosen": -142.03079223632812, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -288.2078552246094, + "loss": 1.0721, + "margin_dpo/margin_mean": 106.0521240234375, + "margin_dpo/margin_std": 131.50982666015625, + "step": 188 + }, + { + "KL/chosen_KL_mean": -106.87858581542969, + "KL/mean": -150.22816467285156, + "KL/rejected_KL_mean": -193.57772827148438, + "KL/std": 118.46228790283203, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.004046589136123657, + "fcm_dpo/delta": 0.05469498038291931, + "fcm_dpo/margin": 86.69915771484375, + "fcm_dpo/q_t": 0.4228058457374573, + "grad_norm": 19.78821563720703, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.3165949285030365, + "logits/rejected": -0.29821348190307617, + "logps/chosen": -159.2775421142578, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -265.7450866699219, + "loss": 1.1441, + "margin_dpo/margin_mean": 86.69915771484375, + "margin_dpo/margin_std": 146.5599365234375, + "step": 189 + }, + { + "KL/chosen_KL_mean": -113.72479248046875, + "KL/mean": -165.66287231445312, + "KL/rejected_KL_mean": -217.60092163085938, + "KL/std": 127.76533508300781, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.0041097975336015224, + "fcm_dpo/delta": -0.034398116171360016, + "fcm_dpo/margin": 103.87614440917969, + "fcm_dpo/q_t": 0.403812050819397, + "grad_norm": 17.153371810913086, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.35712742805480957, + "logits/rejected": -0.3467486500740051, + "logps/chosen": -178.40785217285156, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -320.1514587402344, + "loss": 1.0964, + "margin_dpo/margin_mean": 103.87614440917969, + "margin_dpo/margin_std": 147.73031616210938, + "step": 190 + }, + { + "KL/chosen_KL_mean": -106.41766357421875, + "KL/mean": -180.15927124023438, + "KL/rejected_KL_mean": -253.90084838867188, + "KL/std": 145.85995483398438, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.0036477288231253624, + "fcm_dpo/delta": -0.16866521537303925, + "fcm_dpo/margin": 147.48318481445312, + "fcm_dpo/q_t": 0.3744538426399231, + "grad_norm": 20.244131088256836, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.335135817527771, + "logits/rejected": -0.3109908103942871, + "logps/chosen": -175.0765380859375, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -364.04052734375, + "loss": 0.9946, + "margin_dpo/margin_mean": 147.48318481445312, + "margin_dpo/margin_std": 160.77923583984375, + "step": 191 + }, + { + "KL/chosen_KL_mean": -135.5457000732422, + "KL/mean": -184.35235595703125, + "KL/rejected_KL_mean": -233.15899658203125, + "KL/std": 127.1226806640625, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.0036976111587136984, + "fcm_dpo/delta": 0.04312748461961746, + "fcm_dpo/margin": 97.61331176757812, + "fcm_dpo/q_t": 0.4194992780685425, + "grad_norm": 18.689773559570312, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.3039934039115906, + "logits/rejected": -0.30234575271606445, + "logps/chosen": -205.27261352539062, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -336.4803466796875, + "loss": 1.1342, + "margin_dpo/margin_mean": 97.61331176757812, + "margin_dpo/margin_std": 156.3799591064453, + "step": 192 + }, + { + "KL/chosen_KL_mean": -136.0496826171875, + "KL/mean": -165.52725219726562, + "KL/rejected_KL_mean": -195.00482177734375, + "KL/std": 119.35639190673828, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.0037548099644482136, + "fcm_dpo/delta": 0.052024200558662415, + "fcm_dpo/margin": 58.95512771606445, + "fcm_dpo/q_t": 0.447823703289032, + "grad_norm": 20.45661735534668, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.29755693674087524, + "logits/rejected": -0.2853686809539795, + "logps/chosen": -196.2401885986328, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -271.4123840332031, + "loss": 1.2547, + "margin_dpo/margin_mean": 58.95512771606445, + "margin_dpo/margin_std": 151.63739013671875, + "step": 193 + }, + { + "KL/chosen_KL_mean": -85.49365234375, + "KL/mean": -131.664306640625, + "KL/rejected_KL_mean": -177.8349609375, + "KL/std": 98.282470703125, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.003862201701849699, + "fcm_dpo/delta": 0.048432160168886185, + "fcm_dpo/margin": 92.34130859375, + "fcm_dpo/q_t": 0.4168154001235962, + "grad_norm": 17.16913604736328, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.23439213633537292, + "logits/rejected": -0.21759793162345886, + "logps/chosen": -123.33403015136719, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -238.51974487304688, + "loss": 1.1046, + "margin_dpo/margin_mean": 92.34130859375, + "margin_dpo/margin_std": 108.50004577636719, + "step": 194 + }, + { + "KL/chosen_KL_mean": -134.20651245117188, + "KL/mean": -182.88348388671875, + "KL/rejected_KL_mean": -231.5604248046875, + "KL/std": 120.47825622558594, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.003953356295824051, + "fcm_dpo/delta": 0.017107073217630386, + "fcm_dpo/margin": 97.3539047241211, + "fcm_dpo/q_t": 0.4112229347229004, + "grad_norm": 24.659236907958984, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.2797316610813141, + "logits/rejected": -0.29874980449676514, + "logps/chosen": -189.09808349609375, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -328.3313903808594, + "loss": 1.094, + "margin_dpo/margin_mean": 97.3539047241211, + "margin_dpo/margin_std": 124.069580078125, + "step": 195 + }, + { + "KL/chosen_KL_mean": -106.80827331542969, + "KL/mean": -162.22244262695312, + "KL/rejected_KL_mean": -217.63661193847656, + "KL/std": 122.63679504394531, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.00384792173281312, + "fcm_dpo/delta": -0.03269246220588684, + "fcm_dpo/margin": 110.82833862304688, + "fcm_dpo/q_t": 0.40297311544418335, + "grad_norm": 17.518177032470703, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.29879558086395264, + "logits/rejected": -0.2825039029121399, + "logps/chosen": -160.05352783203125, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -293.6895751953125, + "loss": 1.0839, + "margin_dpo/margin_mean": 110.82833099365234, + "margin_dpo/margin_std": 147.71800231933594, + "step": 196 + }, + { + "KL/chosen_KL_mean": -107.39094543457031, + "KL/mean": -150.92520141601562, + "KL/rejected_KL_mean": -194.45945739746094, + "KL/std": 105.97932434082031, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.0040262117981910706, + "fcm_dpo/delta": 0.053847186267375946, + "fcm_dpo/margin": 87.06851959228516, + "fcm_dpo/q_t": 0.42033201456069946, + "grad_norm": 19.45017433166504, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.2815936803817749, + "logits/rejected": -0.2746492624282837, + "logps/chosen": -167.811279296875, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -271.6683654785156, + "loss": 1.1232, + "margin_dpo/margin_mean": 87.06851959228516, + "margin_dpo/margin_std": 121.93867492675781, + "step": 197 + }, + { + "KL/chosen_KL_mean": -120.8916015625, + "KL/mean": -176.95494079589844, + "KL/rejected_KL_mean": -233.0182647705078, + "KL/std": 134.41259765625, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.003938601817935705, + "fcm_dpo/delta": -0.047928497195243835, + "fcm_dpo/margin": 112.12667846679688, + "fcm_dpo/q_t": 0.40124011039733887, + "grad_norm": 18.25979232788086, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.3253025412559509, + "logits/rejected": -0.32410699129104614, + "logps/chosen": -175.92779541015625, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -330.26153564453125, + "loss": 1.0733, + "margin_dpo/margin_mean": 112.12667846679688, + "margin_dpo/margin_std": 153.6939697265625, + "step": 198 + }, + { + "KL/chosen_KL_mean": -117.14642333984375, + "KL/mean": -173.02886962890625, + "KL/rejected_KL_mean": -228.9113311767578, + "KL/std": 121.26567840576172, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.00379006564617157, + "fcm_dpo/delta": -0.02946564182639122, + "fcm_dpo/margin": 111.76492309570312, + "fcm_dpo/q_t": 0.4028562605381012, + "grad_norm": 18.01993751525879, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.2928391695022583, + "logits/rejected": -0.2746196687221527, + "logps/chosen": -173.97525024414062, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -313.5595397949219, + "loss": 1.085, + "margin_dpo/margin_mean": 111.76492309570312, + "margin_dpo/margin_std": 150.9051513671875, + "step": 199 + }, + { + "KL/chosen_KL_mean": -115.1148681640625, + "KL/mean": -171.5103759765625, + "KL/rejected_KL_mean": -227.90585327148438, + "KL/std": 132.4918670654297, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.0037519584875553846, + "fcm_dpo/delta": -0.027505118399858475, + "fcm_dpo/margin": 112.79100036621094, + "fcm_dpo/q_t": 0.4051932394504547, + "grad_norm": 23.21533966064453, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.2742305397987366, + "logits/rejected": -0.2580034136772156, + "logps/chosen": -168.1819305419922, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -308.5142822265625, + "loss": 1.1009, + "margin_dpo/margin_mean": 112.79100036621094, + "margin_dpo/margin_std": 170.83901977539062, + "step": 200 + }, + { + "KL/chosen_KL_mean": -122.94937133789062, + "KL/mean": -178.7598876953125, + "KL/rejected_KL_mean": -234.5704345703125, + "KL/std": 141.06809997558594, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.003766070818528533, + "fcm_dpo/delta": -0.02365373633801937, + "fcm_dpo/margin": 111.62104797363281, + "fcm_dpo/q_t": 0.4042976498603821, + "grad_norm": 18.088035583496094, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.3276309370994568, + "logits/rejected": -0.3240124583244324, + "logps/chosen": -198.35159301757812, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -349.378662109375, + "loss": 1.0935, + "margin_dpo/margin_mean": 111.62104797363281, + "margin_dpo/margin_std": 158.86087036132812, + "step": 201 + }, + { + "KL/chosen_KL_mean": -126.4018783569336, + "KL/mean": -164.84750366210938, + "KL/rejected_KL_mean": -203.2930908203125, + "KL/std": 122.3680419921875, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.003917068243026733, + "fcm_dpo/delta": 0.10808487236499786, + "fcm_dpo/margin": 76.89122772216797, + "fcm_dpo/q_t": 0.43431201577186584, + "grad_norm": 20.40315055847168, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.27575695514678955, + "logits/rejected": -0.291267991065979, + "logps/chosen": -176.50320434570312, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -290.27813720703125, + "loss": 1.1976, + "margin_dpo/margin_mean": 76.89122772216797, + "margin_dpo/margin_std": 159.47695922851562, + "step": 202 + }, + { + "KL/chosen_KL_mean": -128.38890075683594, + "KL/mean": -177.34750366210938, + "KL/rejected_KL_mean": -226.30609130859375, + "KL/std": 128.1749725341797, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.0039765299297869205, + "fcm_dpo/delta": 0.012002304196357727, + "fcm_dpo/margin": 97.91722106933594, + "fcm_dpo/q_t": 0.4118465185165405, + "grad_norm": 20.225921630859375, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.32165127992630005, + "logits/rejected": -0.3128536343574524, + "logps/chosen": -188.99859619140625, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -312.20208740234375, + "loss": 1.1079, + "margin_dpo/margin_mean": 97.9172134399414, + "margin_dpo/margin_std": 142.193115234375, + "step": 203 + }, + { + "KL/chosen_KL_mean": -140.11990356445312, + "KL/mean": -184.9493865966797, + "KL/rejected_KL_mean": -229.77886962890625, + "KL/std": 138.39321899414062, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.004067155998200178, + "fcm_dpo/delta": 0.03968627005815506, + "fcm_dpo/margin": 89.65896606445312, + "fcm_dpo/q_t": 0.4207793176174164, + "grad_norm": 23.430374145507812, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.4037870168685913, + "logits/rejected": -0.3659971356391907, + "logps/chosen": -220.2848663330078, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -317.47479248046875, + "loss": 1.1569, + "margin_dpo/margin_mean": 89.65896606445312, + "margin_dpo/margin_std": 163.48477172851562, + "step": 204 + }, + { + "KL/chosen_KL_mean": -134.97842407226562, + "KL/mean": -193.99037170410156, + "KL/rejected_KL_mean": -253.00233459472656, + "KL/std": 138.9073486328125, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.0038606300950050354, + "fcm_dpo/delta": -0.07167132198810577, + "fcm_dpo/margin": 118.0239028930664, + "fcm_dpo/q_t": 0.39341607689857483, + "grad_norm": 20.353437423706055, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.30310916900634766, + "logits/rejected": -0.2724456191062927, + "logps/chosen": -194.3631591796875, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -338.12738037109375, + "loss": 1.071, + "margin_dpo/margin_mean": 118.02389526367188, + "margin_dpo/margin_std": 153.8232421875, + "step": 205 + }, + { + "KL/chosen_KL_mean": -121.64984893798828, + "KL/mean": -186.41314697265625, + "KL/rejected_KL_mean": -251.1764678955078, + "KL/std": 126.77278137207031, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.003710696939378977, + "fcm_dpo/delta": -0.09660979360342026, + "fcm_dpo/margin": 129.526611328125, + "fcm_dpo/q_t": 0.3851868808269501, + "grad_norm": 20.32840347290039, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.2523133158683777, + "logits/rejected": -0.25322118401527405, + "logps/chosen": -168.61434936523438, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -350.12994384765625, + "loss": 1.0251, + "margin_dpo/margin_mean": 129.526611328125, + "margin_dpo/margin_std": 138.80824279785156, + "step": 206 + }, + { + "KL/chosen_KL_mean": -114.11463165283203, + "KL/mean": -186.99752807617188, + "KL/rejected_KL_mean": -259.88043212890625, + "KL/std": 146.8643798828125, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.0034998529590666294, + "fcm_dpo/delta": -0.12993571162223816, + "fcm_dpo/margin": 145.76577758789062, + "fcm_dpo/q_t": 0.38310253620147705, + "grad_norm": 20.39961814880371, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.33750367164611816, + "logits/rejected": -0.3120267987251282, + "logps/chosen": -170.1708984375, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -344.3282165527344, + "loss": 1.0176, + "margin_dpo/margin_mean": 145.76577758789062, + "margin_dpo/margin_std": 169.3638153076172, + "step": 207 + }, + { + "KL/chosen_KL_mean": -173.23487854003906, + "KL/mean": -231.67062377929688, + "KL/rejected_KL_mean": -290.10638427734375, + "KL/std": 143.04248046875, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.0034020745661109686, + "fcm_dpo/delta": 0.0023834500461816788, + "fcm_dpo/margin": 116.8714828491211, + "fcm_dpo/q_t": 0.40898996591567993, + "grad_norm": 28.37578773498535, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.35492682456970215, + "logits/rejected": -0.33707547187805176, + "logps/chosen": -240.302490234375, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -384.3932800292969, + "loss": 1.1087, + "margin_dpo/margin_mean": 116.8714828491211, + "margin_dpo/margin_std": 172.8236083984375, + "step": 208 + }, + { + "KL/chosen_KL_mean": -145.79742431640625, + "KL/mean": -196.9249267578125, + "KL/rejected_KL_mean": -248.0524139404297, + "KL/std": 129.4995574951172, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.0035100283566862345, + "fcm_dpo/delta": 0.04577942192554474, + "fcm_dpo/margin": 102.25497436523438, + "fcm_dpo/q_t": 0.4185419976711273, + "grad_norm": 23.225322723388672, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.24873411655426025, + "logits/rejected": -0.23527741432189941, + "logps/chosen": -201.9791259765625, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -328.9939270019531, + "loss": 1.1363, + "margin_dpo/margin_mean": 102.25497436523438, + "margin_dpo/margin_std": 160.52923583984375, + "step": 209 + }, + { + "KL/chosen_KL_mean": -129.65380859375, + "KL/mean": -188.928466796875, + "KL/rejected_KL_mean": -248.20309448242188, + "KL/std": 131.33786010742188, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.003531618043780327, + "fcm_dpo/delta": -0.022135471925139427, + "fcm_dpo/margin": 118.54928588867188, + "fcm_dpo/q_t": 0.40433353185653687, + "grad_norm": 22.45277214050293, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.2742435038089752, + "logits/rejected": -0.2655297815799713, + "logps/chosen": -176.025634765625, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -324.88470458984375, + "loss": 1.0842, + "margin_dpo/margin_mean": 118.5492935180664, + "margin_dpo/margin_std": 157.50991821289062, + "step": 210 + }, + { + "KL/chosen_KL_mean": -189.68521118164062, + "KL/mean": -239.61474609375, + "KL/rejected_KL_mean": -289.5442810058594, + "KL/std": 151.8519744873047, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.0035626532044261694, + "fcm_dpo/delta": 0.04933279752731323, + "fcm_dpo/margin": 99.85906982421875, + "fcm_dpo/q_t": 0.42364493012428284, + "grad_norm": 35.19536590576172, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.28093987703323364, + "logits/rejected": -0.24116767942905426, + "logps/chosen": -268.6175537109375, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -376.3652648925781, + "loss": 1.1573, + "margin_dpo/margin_mean": 99.85906982421875, + "margin_dpo/margin_std": 178.33770751953125, + "step": 211 + }, + { + "KL/chosen_KL_mean": -156.86328125, + "KL/mean": -228.39266967773438, + "KL/rejected_KL_mean": -299.92205810546875, + "KL/std": 160.87054443359375, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.003351601306349039, + "fcm_dpo/delta": -0.09883411228656769, + "fcm_dpo/margin": 143.05877685546875, + "fcm_dpo/q_t": 0.389013409614563, + "grad_norm": 23.036394119262695, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.2952839732170105, + "logits/rejected": -0.2883192002773285, + "logps/chosen": -215.060302734375, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -402.97991943359375, + "loss": 1.0537, + "margin_dpo/margin_mean": 143.05877685546875, + "margin_dpo/margin_std": 179.50482177734375, + "step": 212 + }, + { + "KL/chosen_KL_mean": -147.60040283203125, + "KL/mean": -217.38333129882812, + "KL/rejected_KL_mean": -287.166259765625, + "KL/std": 143.98876953125, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.00324892345815897, + "fcm_dpo/delta": -0.06323815882205963, + "fcm_dpo/margin": 139.56590270996094, + "fcm_dpo/q_t": 0.3945580720901489, + "grad_norm": 25.17753791809082, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.3027662932872772, + "logits/rejected": -0.27730774879455566, + "logps/chosen": -215.11309814453125, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -381.08099365234375, + "loss": 1.0547, + "margin_dpo/margin_mean": 139.56588745117188, + "margin_dpo/margin_std": 169.9403839111328, + "step": 213 + }, + { + "KL/chosen_KL_mean": -129.4480438232422, + "KL/mean": -192.62973022460938, + "KL/rejected_KL_mean": -255.8114013671875, + "KL/std": 141.9759521484375, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.003236332442611456, + "fcm_dpo/delta": -0.010202363133430481, + "fcm_dpo/margin": 126.36334991455078, + "fcm_dpo/q_t": 0.40674763917922974, + "grad_norm": 22.835277557373047, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.23017573356628418, + "logits/rejected": -0.23523810505867004, + "logps/chosen": -171.0529327392578, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -333.32879638671875, + "loss": 1.0857, + "margin_dpo/margin_mean": 126.36334991455078, + "margin_dpo/margin_std": 166.39173889160156, + "step": 214 + }, + { + "KL/chosen_KL_mean": -150.3916473388672, + "KL/mean": -218.00868225097656, + "KL/rejected_KL_mean": -285.625732421875, + "KL/std": 149.27084350585938, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.00315915048122406, + "fcm_dpo/delta": -0.03191942721605301, + "fcm_dpo/margin": 135.23406982421875, + "fcm_dpo/q_t": 0.40022581815719604, + "grad_norm": 23.505481719970703, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.2642921805381775, + "logits/rejected": -0.2557499408721924, + "logps/chosen": -203.67091369628906, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -375.5903625488281, + "loss": 1.0594, + "margin_dpo/margin_mean": 135.23406982421875, + "margin_dpo/margin_std": 153.5160675048828, + "step": 215 + }, + { + "KL/chosen_KL_mean": -155.68069458007812, + "KL/mean": -220.76373291015625, + "KL/rejected_KL_mean": -285.8468017578125, + "KL/std": 153.577392578125, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.0031813611276447773, + "fcm_dpo/delta": -0.016869423910975456, + "fcm_dpo/margin": 130.16610717773438, + "fcm_dpo/q_t": 0.40590834617614746, + "grad_norm": 24.740102767944336, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.2661086320877075, + "logits/rejected": -0.2514107823371887, + "logps/chosen": -204.5684814453125, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -363.04571533203125, + "loss": 1.1001, + "margin_dpo/margin_mean": 130.16610717773438, + "margin_dpo/margin_std": 189.28089904785156, + "step": 216 + }, + { + "KL/chosen_KL_mean": -162.05545043945312, + "KL/mean": -239.3953857421875, + "KL/rejected_KL_mean": -316.7353515625, + "KL/std": 158.39334106445312, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.003011965425685048, + "fcm_dpo/delta": -0.0773845985531807, + "fcm_dpo/margin": 154.67990112304688, + "fcm_dpo/q_t": 0.39230814576148987, + "grad_norm": 19.440380096435547, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.2700842022895813, + "logits/rejected": -0.26593339443206787, + "logps/chosen": -211.9007568359375, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -416.81365966796875, + "loss": 1.0411, + "margin_dpo/margin_mean": 154.6798858642578, + "margin_dpo/margin_std": 180.6573944091797, + "step": 217 + }, + { + "KL/chosen_KL_mean": -169.6825714111328, + "KL/mean": -228.77969360351562, + "KL/rejected_KL_mean": -287.8768310546875, + "KL/std": 157.65548706054688, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.0030449300538748503, + "fcm_dpo/delta": 0.04503227025270462, + "fcm_dpo/margin": 118.19425201416016, + "fcm_dpo/q_t": 0.4195507764816284, + "grad_norm": 21.25322151184082, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.25268301367759705, + "logits/rejected": -0.24772432446479797, + "logps/chosen": -228.25924682617188, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -375.72320556640625, + "loss": 1.1352, + "margin_dpo/margin_mean": 118.19425964355469, + "margin_dpo/margin_std": 186.68544006347656, + "step": 218 + }, + { + "KL/chosen_KL_mean": -182.89511108398438, + "KL/mean": -232.8427276611328, + "KL/rejected_KL_mean": -282.79034423828125, + "KL/std": 163.54734802246094, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.0030733770690858364, + "fcm_dpo/delta": -0.002437241841107607, + "fcm_dpo/margin": 99.89521789550781, + "fcm_dpo/q_t": 0.4305492639541626, + "grad_norm": 28.50415802001953, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.2840089201927185, + "logits/rejected": -0.2790279984474182, + "logps/chosen": -243.97897338867188, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -368.62078857421875, + "loss": 1.1944, + "margin_dpo/margin_mean": 99.89521789550781, + "margin_dpo/margin_std": 201.90310668945312, + "step": 219 + }, + { + "KL/chosen_KL_mean": -206.1656494140625, + "KL/mean": -247.63449096679688, + "KL/rejected_KL_mean": -289.10333251953125, + "KL/std": 140.86788940429688, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.0032577612437307835, + "fcm_dpo/delta": 0.1417321115732193, + "fcm_dpo/margin": 82.93766784667969, + "fcm_dpo/q_t": 0.43858277797698975, + "grad_norm": 26.906091690063477, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.2821032702922821, + "logits/rejected": -0.2597326636314392, + "logps/chosen": -276.19696044921875, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -376.7888488769531, + "loss": 1.1887, + "margin_dpo/margin_mean": 82.93767547607422, + "margin_dpo/margin_std": 145.25137329101562, + "step": 220 + }, + { + "KL/chosen_KL_mean": -171.998046875, + "KL/mean": -268.70098876953125, + "KL/rejected_KL_mean": -365.4039611816406, + "KL/std": 178.30426025390625, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.0030434958171099424, + "fcm_dpo/delta": -0.22819769382476807, + "fcm_dpo/margin": 193.40589904785156, + "fcm_dpo/q_t": 0.3597332835197449, + "grad_norm": 27.25739288330078, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.25868871808052063, + "logits/rejected": -0.26574745774269104, + "logps/chosen": -224.15277099609375, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -471.87164306640625, + "loss": 0.9579, + "margin_dpo/margin_mean": 193.40589904785156, + "margin_dpo/margin_std": 184.6766815185547, + "step": 221 + }, + { + "KL/chosen_KL_mean": -177.10008239746094, + "KL/mean": -247.15863037109375, + "KL/rejected_KL_mean": -317.2171630859375, + "KL/std": 162.72874450683594, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.0029336886946111917, + "fcm_dpo/delta": -0.013469204306602478, + "fcm_dpo/margin": 140.11708068847656, + "fcm_dpo/q_t": 0.40566956996917725, + "grad_norm": 17.6011905670166, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.32022494077682495, + "logits/rejected": -0.3259966969490051, + "logps/chosen": -238.07118225097656, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -417.21832275390625, + "loss": 1.0861, + "margin_dpo/margin_mean": 140.11708068847656, + "margin_dpo/margin_std": 184.19187927246094, + "step": 222 + }, + { + "KL/chosen_KL_mean": -184.8544921875, + "KL/mean": -244.89614868164062, + "KL/rejected_KL_mean": -304.93780517578125, + "KL/std": 158.01258850097656, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.002949609886854887, + "fcm_dpo/delta": 0.051337942481040955, + "fcm_dpo/margin": 120.08331298828125, + "fcm_dpo/q_t": 0.4204061031341553, + "grad_norm": 23.18364715576172, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.2701888978481293, + "logits/rejected": -0.267690509557724, + "logps/chosen": -237.4950714111328, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -387.7628479003906, + "loss": 1.1449, + "margin_dpo/margin_mean": 120.08332824707031, + "margin_dpo/margin_std": 200.96475219726562, + "step": 223 + }, + { + "KL/chosen_KL_mean": -160.8324432373047, + "KL/mean": -227.91595458984375, + "KL/rejected_KL_mean": -294.99951171875, + "KL/std": 173.14578247070312, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.0029319876339286566, + "fcm_dpo/delta": 0.004883028566837311, + "fcm_dpo/margin": 134.1670379638672, + "fcm_dpo/q_t": 0.41187167167663574, + "grad_norm": 21.665334701538086, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.30513614416122437, + "logits/rejected": -0.2876437306404114, + "logps/chosen": -209.4278564453125, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -372.1159973144531, + "loss": 1.1137, + "margin_dpo/margin_mean": 134.1670379638672, + "margin_dpo/margin_std": 200.36749267578125, + "step": 224 + }, + { + "KL/chosen_KL_mean": -178.73651123046875, + "KL/mean": -257.0627136230469, + "KL/rejected_KL_mean": -335.388916015625, + "KL/std": 163.3724365234375, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.0029020393267273903, + "fcm_dpo/delta": -0.06344681233167648, + "fcm_dpo/margin": 156.65240478515625, + "fcm_dpo/q_t": 0.39548754692077637, + "grad_norm": 19.82950782775879, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.314418226480484, + "logits/rejected": -0.304479718208313, + "logps/chosen": -236.73696899414062, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -435.2918395996094, + "loss": 1.0593, + "margin_dpo/margin_mean": 156.65240478515625, + "margin_dpo/margin_std": 200.25833129882812, + "step": 225 + }, + { + "KL/chosen_KL_mean": -151.73455810546875, + "KL/mean": -212.6505889892578, + "KL/rejected_KL_mean": -273.5666198730469, + "KL/std": 156.52337646484375, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.002900505904108286, + "fcm_dpo/delta": 0.051205385476350784, + "fcm_dpo/margin": 121.83206939697266, + "fcm_dpo/q_t": 0.42080676555633545, + "grad_norm": 24.11922264099121, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.3567095994949341, + "logits/rejected": -0.31852173805236816, + "logps/chosen": -210.63336181640625, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -352.25439453125, + "loss": 1.1319, + "margin_dpo/margin_mean": 121.83206939697266, + "margin_dpo/margin_std": 181.67535400390625, + "step": 226 + }, + { + "KL/chosen_KL_mean": -174.72982788085938, + "KL/mean": -252.92881774902344, + "KL/rejected_KL_mean": -331.1278076171875, + "KL/std": 179.85467529296875, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.0028641903772950172, + "fcm_dpo/delta": -0.056932441890239716, + "fcm_dpo/margin": 156.39801025390625, + "fcm_dpo/q_t": 0.39597082138061523, + "grad_norm": 19.867029190063477, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.3615909814834595, + "logits/rejected": -0.3510950207710266, + "logps/chosen": -233.802001953125, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -430.54022216796875, + "loss": 1.0543, + "margin_dpo/margin_mean": 156.39801025390625, + "margin_dpo/margin_std": 188.0986328125, + "step": 227 + }, + { + "KL/chosen_KL_mean": -181.21298217773438, + "KL/mean": -235.09324645996094, + "KL/rejected_KL_mean": -288.9735107421875, + "KL/std": 147.68833923339844, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.002995947375893593, + "fcm_dpo/delta": 0.08476278185844421, + "fcm_dpo/margin": 107.76055908203125, + "fcm_dpo/q_t": 0.42690229415893555, + "grad_norm": 21.91815185546875, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.34226393699645996, + "logits/rejected": -0.3274916112422943, + "logps/chosen": -247.10426330566406, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -380.02227783203125, + "loss": 1.1607, + "margin_dpo/margin_mean": 107.76055908203125, + "margin_dpo/margin_std": 183.66375732421875, + "step": 228 + }, + { + "KL/chosen_KL_mean": -188.99282836914062, + "KL/mean": -245.96768188476562, + "KL/rejected_KL_mean": -302.9425048828125, + "KL/std": 172.01333618164062, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.0030979858711361885, + "fcm_dpo/delta": 0.052381522953510284, + "fcm_dpo/margin": 113.94969177246094, + "fcm_dpo/q_t": 0.4193553626537323, + "grad_norm": 31.544322967529297, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.3947259187698364, + "logits/rejected": -0.3649597764015198, + "logps/chosen": -259.69921875, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -387.46990966796875, + "loss": 1.1342, + "margin_dpo/margin_mean": 113.9496841430664, + "margin_dpo/margin_std": 177.01577758789062, + "step": 229 + }, + { + "KL/chosen_KL_mean": -146.11764526367188, + "KL/mean": -236.42767333984375, + "KL/rejected_KL_mean": -326.7377014160156, + "KL/std": 162.74221801757812, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.0029000695794820786, + "fcm_dpo/delta": -0.14832410216331482, + "fcm_dpo/margin": 180.6200408935547, + "fcm_dpo/q_t": 0.37458282709121704, + "grad_norm": 27.4748592376709, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.294960618019104, + "logits/rejected": -0.3019281327724457, + "logps/chosen": -185.399658203125, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -412.359619140625, + "loss": 0.9909, + "margin_dpo/margin_mean": 180.6200408935547, + "margin_dpo/margin_std": 177.76747131347656, + "step": 230 + }, + { + "KL/chosen_KL_mean": -168.8487091064453, + "KL/mean": -230.40908813476562, + "KL/rejected_KL_mean": -291.96942138671875, + "KL/std": 143.62872314453125, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.00291502196341753, + "fcm_dpo/delta": 0.04594101011753082, + "fcm_dpo/margin": 123.12074279785156, + "fcm_dpo/q_t": 0.4190780222415924, + "grad_norm": 22.074596405029297, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.3602941334247589, + "logits/rejected": -0.33326324820518494, + "logps/chosen": -232.12515258789062, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -366.09332275390625, + "loss": 1.117, + "margin_dpo/margin_mean": 123.12074279785156, + "margin_dpo/margin_std": 170.6857452392578, + "step": 231 + }, + { + "KL/chosen_KL_mean": -205.226806640625, + "KL/mean": -255.25827026367188, + "KL/rejected_KL_mean": -305.28973388671875, + "KL/std": 170.07846069335938, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.0030578605365008116, + "fcm_dpo/delta": 0.10380719602108002, + "fcm_dpo/margin": 100.06292724609375, + "fcm_dpo/q_t": 0.4334006905555725, + "grad_norm": 23.328777313232422, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.39860862493515015, + "logits/rejected": -0.3822248876094818, + "logps/chosen": -275.9755859375, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -389.2668151855469, + "loss": 1.1725, + "margin_dpo/margin_mean": 100.06293487548828, + "margin_dpo/margin_std": 179.21624755859375, + "step": 232 + }, + { + "KL/chosen_KL_mean": -180.83973693847656, + "KL/mean": -261.960205078125, + "KL/rejected_KL_mean": -343.0806579589844, + "KL/std": 182.64207458496094, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.00300042936578393, + "fcm_dpo/delta": -0.10129574686288834, + "fcm_dpo/margin": 162.24093627929688, + "fcm_dpo/q_t": 0.39332348108291626, + "grad_norm": 27.694826126098633, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.3419656753540039, + "logits/rejected": -0.3451586961746216, + "logps/chosen": -235.72267150878906, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -450.56072998046875, + "loss": 1.0748, + "margin_dpo/margin_mean": 162.24093627929688, + "margin_dpo/margin_std": 240.9387664794922, + "step": 233 + }, + { + "KL/chosen_KL_mean": -175.83416748046875, + "KL/mean": -258.9500732421875, + "KL/rejected_KL_mean": -342.06597900390625, + "KL/std": 163.5806884765625, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.0027665914967656136, + "fcm_dpo/delta": -0.07618995010852814, + "fcm_dpo/margin": 166.2318115234375, + "fcm_dpo/q_t": 0.3922733962535858, + "grad_norm": 25.569175720214844, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.315540075302124, + "logits/rejected": -0.3419063985347748, + "logps/chosen": -219.92868041992188, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -442.0726318359375, + "loss": 1.0445, + "margin_dpo/margin_mean": 166.23182678222656, + "margin_dpo/margin_std": 185.68582153320312, + "step": 234 + }, + { + "KL/chosen_KL_mean": -207.25558471679688, + "KL/mean": -258.892333984375, + "KL/rejected_KL_mean": -310.5290832519531, + "KL/std": 150.97354125976562, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0029631350189447403, + "fcm_dpo/delta": 0.10171176493167877, + "fcm_dpo/margin": 103.27351379394531, + "fcm_dpo/q_t": 0.43064451217651367, + "grad_norm": 24.898860931396484, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.376761257648468, + "logits/rejected": -0.3584543466567993, + "logps/chosen": -269.4934997558594, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -400.92413330078125, + "loss": 1.1756, + "margin_dpo/margin_mean": 103.27351379394531, + "margin_dpo/margin_std": 185.69923400878906, + "step": 235 + }, + { + "KL/chosen_KL_mean": -146.16514587402344, + "KL/mean": -230.15301513671875, + "KL/rejected_KL_mean": -314.1408996582031, + "KL/std": 158.77590942382812, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.002861781744286418, + "fcm_dpo/delta": -0.09425194561481476, + "fcm_dpo/margin": 167.97573852539062, + "fcm_dpo/q_t": 0.3849557042121887, + "grad_norm": 47.46019744873047, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.3816717565059662, + "logits/rejected": -0.38343510031700134, + "logps/chosen": -195.50650024414062, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -417.65252685546875, + "loss": 1.0082, + "margin_dpo/margin_mean": 167.97573852539062, + "margin_dpo/margin_std": 153.7233123779297, + "step": 236 + }, + { + "KL/chosen_KL_mean": -202.48997497558594, + "KL/mean": -264.41650390625, + "KL/rejected_KL_mean": -326.34307861328125, + "KL/std": 159.1524658203125, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0028853025287389755, + "fcm_dpo/delta": 0.04754340276122093, + "fcm_dpo/margin": 123.85310363769531, + "fcm_dpo/q_t": 0.41874706745147705, + "grad_norm": 29.721525192260742, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.38273727893829346, + "logits/rejected": -0.3775549530982971, + "logps/chosen": -256.6580810546875, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -421.12347412109375, + "loss": 1.1258, + "margin_dpo/margin_mean": 123.85310363769531, + "margin_dpo/margin_std": 184.06011962890625, + "step": 237 + }, + { + "KL/chosen_KL_mean": -181.52700805664062, + "KL/mean": -245.96737670898438, + "KL/rejected_KL_mean": -310.40777587890625, + "KL/std": 159.92538452148438, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0028985640965402126, + "fcm_dpo/delta": 0.028993351384997368, + "fcm_dpo/margin": 128.88072204589844, + "fcm_dpo/q_t": 0.416244238615036, + "grad_norm": 24.590639114379883, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.3403720259666443, + "logits/rejected": -0.35210222005844116, + "logps/chosen": -235.50013732910156, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -399.82568359375, + "loss": 1.1103, + "margin_dpo/margin_mean": 128.88072204589844, + "margin_dpo/margin_std": 179.00254821777344, + "step": 238 + }, + { + "KL/chosen_KL_mean": -196.514892578125, + "KL/mean": -269.5975036621094, + "KL/rejected_KL_mean": -342.6800842285156, + "KL/std": 152.7589111328125, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.0028616369236260653, + "fcm_dpo/delta": -0.024076886475086212, + "fcm_dpo/margin": 146.16522216796875, + "fcm_dpo/q_t": 0.401398241519928, + "grad_norm": 28.642642974853516, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.36739569902420044, + "logits/rejected": -0.35801225900650024, + "logps/chosen": -254.61270141601562, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -436.2730407714844, + "loss": 1.0701, + "margin_dpo/margin_mean": 146.16522216796875, + "margin_dpo/margin_std": 168.28749084472656, + "step": 239 + }, + { + "KL/chosen_KL_mean": -212.81607055664062, + "KL/mean": -272.19439697265625, + "KL/rejected_KL_mean": -331.57269287109375, + "KL/std": 167.1009521484375, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.0029481430537998676, + "fcm_dpo/delta": 0.05555122345685959, + "fcm_dpo/margin": 118.75664520263672, + "fcm_dpo/q_t": 0.42097073793411255, + "grad_norm": 33.100982666015625, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.36133328080177307, + "logits/rejected": -0.33984366059303284, + "logps/chosen": -273.4305419921875, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -405.69122314453125, + "loss": 1.1517, + "margin_dpo/margin_mean": 118.75664520263672, + "margin_dpo/margin_std": 199.55770874023438, + "step": 240 + }, + { + "KL/chosen_KL_mean": -183.80712890625, + "KL/mean": -276.44439697265625, + "KL/rejected_KL_mean": -369.0816650390625, + "KL/std": 185.93853759765625, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0028273449279367924, + "fcm_dpo/delta": -0.1465643048286438, + "fcm_dpo/margin": 185.27456665039062, + "fcm_dpo/q_t": 0.37906432151794434, + "grad_norm": 24.62763023376465, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.37756481766700745, + "logits/rejected": -0.35448551177978516, + "logps/chosen": -249.89820861816406, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -457.142578125, + "loss": 1.012, + "margin_dpo/margin_mean": 185.27456665039062, + "margin_dpo/margin_std": 211.22525024414062, + "step": 241 + }, + { + "KL/chosen_KL_mean": -206.30630493164062, + "KL/mean": -273.1295166015625, + "KL/rejected_KL_mean": -339.9527282714844, + "KL/std": 161.26638793945312, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.0028189460281282663, + "fcm_dpo/delta": 0.024674097076058388, + "fcm_dpo/margin": 133.6464385986328, + "fcm_dpo/q_t": 0.4148985743522644, + "grad_norm": 26.225845336914062, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.32095208764076233, + "logits/rejected": -0.2974799871444702, + "logps/chosen": -274.17022705078125, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -423.31304931640625, + "loss": 1.1137, + "margin_dpo/margin_mean": 133.6464385986328, + "margin_dpo/margin_std": 192.44961547851562, + "step": 242 + }, + { + "KL/chosen_KL_mean": -201.5411834716797, + "KL/mean": -283.2385559082031, + "KL/rejected_KL_mean": -364.9359130859375, + "KL/std": 177.864013671875, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.002721097320318222, + "fcm_dpo/delta": -0.0517343208193779, + "fcm_dpo/margin": 163.39471435546875, + "fcm_dpo/q_t": 0.3970077633857727, + "grad_norm": 23.662738800048828, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.34067827463150024, + "logits/rejected": -0.31491512060165405, + "logps/chosen": -264.62542724609375, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -441.27154541015625, + "loss": 1.0579, + "margin_dpo/margin_mean": 163.39471435546875, + "margin_dpo/margin_std": 199.94488525390625, + "step": 243 + }, + { + "KL/chosen_KL_mean": -178.40692138671875, + "KL/mean": -265.0865783691406, + "KL/rejected_KL_mean": -351.7662353515625, + "KL/std": 173.98562622070312, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.0026149326004087925, + "fcm_dpo/delta": -0.06256618350744247, + "fcm_dpo/margin": 173.35928344726562, + "fcm_dpo/q_t": 0.39380860328674316, + "grad_norm": 24.746776580810547, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.39362722635269165, + "logits/rejected": -0.3907374441623688, + "logps/chosen": -239.547607421875, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -446.65814208984375, + "loss": 1.0371, + "margin_dpo/margin_mean": 173.35928344726562, + "margin_dpo/margin_std": 181.51531982421875, + "step": 244 + }, + { + "KL/chosen_KL_mean": -206.80569458007812, + "KL/mean": -273.0625915527344, + "KL/rejected_KL_mean": -339.31951904296875, + "KL/std": 168.91696166992188, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0025293505750596523, + "fcm_dpo/delta": -0.05094674602150917, + "fcm_dpo/margin": 132.5137939453125, + "fcm_dpo/q_t": 0.4197985529899597, + "grad_norm": 23.293956756591797, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.39832237362861633, + "logits/rejected": -0.37540921568870544, + "logps/chosen": -274.0679931640625, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -426.9595947265625, + "loss": 1.138, + "margin_dpo/margin_mean": 132.5137939453125, + "margin_dpo/margin_std": 189.8162384033203, + "step": 245 + }, + { + "KL/chosen_KL_mean": -205.90045166015625, + "KL/mean": -273.2760009765625, + "KL/rejected_KL_mean": -340.65155029296875, + "KL/std": 187.2190399169922, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.0025771402288228273, + "fcm_dpo/delta": 0.05890655517578125, + "fcm_dpo/margin": 134.7510986328125, + "fcm_dpo/q_t": 0.42192304134368896, + "grad_norm": 23.917299270629883, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.432862788438797, + "logits/rejected": -0.39796045422554016, + "logps/chosen": -272.597412109375, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -424.9978942871094, + "loss": 1.1311, + "margin_dpo/margin_mean": 134.7510986328125, + "margin_dpo/margin_std": 200.21771240234375, + "step": 246 + }, + { + "KL/chosen_KL_mean": -240.87014770507812, + "KL/mean": -337.90625, + "KL/rejected_KL_mean": -434.9423828125, + "KL/std": 193.46206665039062, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.0024917416740208864, + "fcm_dpo/delta": -0.09881572425365448, + "fcm_dpo/margin": 194.0722198486328, + "fcm_dpo/q_t": 0.38646528124809265, + "grad_norm": 27.314537048339844, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.34786272048950195, + "logits/rejected": -0.3472127914428711, + "logps/chosen": -297.4754943847656, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -541.2356567382812, + "loss": 1.0265, + "margin_dpo/margin_mean": 194.07223510742188, + "margin_dpo/margin_std": 212.13206481933594, + "step": 247 + }, + { + "KL/chosen_KL_mean": -209.49447631835938, + "KL/mean": -293.801513671875, + "KL/rejected_KL_mean": -378.1085510253906, + "KL/std": 159.87808227539062, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.00242455187253654, + "fcm_dpo/delta": -0.011476751416921616, + "fcm_dpo/margin": 168.61410522460938, + "fcm_dpo/q_t": 0.40217316150665283, + "grad_norm": 33.130226135253906, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.36514410376548767, + "logits/rejected": -0.38432562351226807, + "logps/chosen": -253.53768920898438, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -469.9654235839844, + "loss": 1.0555, + "margin_dpo/margin_mean": 168.61410522460938, + "margin_dpo/margin_std": 157.13619995117188, + "step": 248 + }, + { + "KL/chosen_KL_mean": -263.66705322265625, + "KL/mean": -310.262451171875, + "KL/rejected_KL_mean": -356.8578796386719, + "KL/std": 177.94171142578125, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0025667999871075153, + "fcm_dpo/delta": 0.07588154077529907, + "fcm_dpo/margin": 93.19084167480469, + "fcm_dpo/q_t": 0.4462388753890991, + "grad_norm": 38.58940887451172, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.4191044569015503, + "logits/rejected": -0.3978833556175232, + "logps/chosen": -326.109375, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -437.325927734375, + "loss": 1.2345, + "margin_dpo/margin_mean": 93.19084167480469, + "margin_dpo/margin_std": 215.5201873779297, + "step": 249 + }, + { + "KL/chosen_KL_mean": -234.6265869140625, + "KL/mean": -317.1041259765625, + "KL/rejected_KL_mean": -399.5816955566406, + "KL/std": 174.60850524902344, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.0025672472547739744, + "fcm_dpo/delta": -0.027622459456324577, + "fcm_dpo/margin": 164.95510864257812, + "fcm_dpo/q_t": 0.4005199372768402, + "grad_norm": 50.49326705932617, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.4102703928947449, + "logits/rejected": -0.3625056743621826, + "logps/chosen": -300.2632751464844, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -473.45355224609375, + "loss": 1.0625, + "margin_dpo/margin_mean": 164.95510864257812, + "margin_dpo/margin_std": 185.71395874023438, + "step": 250 + }, + { + "KL/chosen_KL_mean": -243.0663604736328, + "KL/mean": -297.49920654296875, + "KL/rejected_KL_mean": -351.93206787109375, + "KL/std": 178.23744201660156, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.0026549724861979485, + "fcm_dpo/delta": 0.12186935544013977, + "fcm_dpo/margin": 108.86573791503906, + "fcm_dpo/q_t": 0.43518325686454773, + "grad_norm": 28.73383331298828, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.3828049302101135, + "logits/rejected": -0.3556620478630066, + "logps/chosen": -300.24908447265625, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -429.59552001953125, + "loss": 1.1871, + "margin_dpo/margin_mean": 108.86573791503906, + "margin_dpo/margin_std": 198.632568359375, + "step": 251 + }, + { + "KL/chosen_KL_mean": -234.64947509765625, + "KL/mean": -310.8502197265625, + "KL/rejected_KL_mean": -387.05096435546875, + "KL/std": 155.4658660888672, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.002685960615053773, + "fcm_dpo/delta": -0.01100852806121111, + "fcm_dpo/margin": 152.40151977539062, + "fcm_dpo/q_t": 0.40194863080978394, + "grad_norm": 28.698516845703125, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.41327089071273804, + "logits/rejected": -0.3738570511341095, + "logps/chosen": -306.3351135253906, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -471.8089599609375, + "loss": 1.0622, + "margin_dpo/margin_mean": 152.40151977539062, + "margin_dpo/margin_std": 158.24240112304688, + "step": 252 + }, + { + "KL/chosen_KL_mean": -198.50045776367188, + "KL/mean": -275.0964660644531, + "KL/rejected_KL_mean": -351.6924743652344, + "KL/std": 165.58140563964844, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0027235562447458506, + "fcm_dpo/delta": -0.02177414670586586, + "fcm_dpo/margin": 153.19198608398438, + "fcm_dpo/q_t": 0.4017912745475769, + "grad_norm": 22.54262924194336, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.43282192945480347, + "logits/rejected": -0.4252409338951111, + "logps/chosen": -267.6343994140625, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -450.39501953125, + "loss": 1.0762, + "margin_dpo/margin_mean": 153.19200134277344, + "margin_dpo/margin_std": 183.40386962890625, + "step": 253 + }, + { + "KL/chosen_KL_mean": -191.62738037109375, + "KL/mean": -257.4107360839844, + "KL/rejected_KL_mean": -323.194091796875, + "KL/std": 173.69381713867188, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.002716494258493185, + "fcm_dpo/delta": 0.047766633331775665, + "fcm_dpo/margin": 131.56671142578125, + "fcm_dpo/q_t": 0.42082470655441284, + "grad_norm": 22.608551025390625, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.38302332162857056, + "logits/rejected": -0.3713380992412567, + "logps/chosen": -245.78237915039062, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -413.5017395019531, + "loss": 1.1391, + "margin_dpo/margin_mean": 131.56671142578125, + "margin_dpo/margin_std": 216.3385009765625, + "step": 254 + }, + { + "KL/chosen_KL_mean": -192.4730224609375, + "KL/mean": -261.29510498046875, + "KL/rejected_KL_mean": -330.1172180175781, + "KL/std": 152.119140625, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.002739853225648403, + "fcm_dpo/delta": 0.025143781676888466, + "fcm_dpo/margin": 137.64418029785156, + "fcm_dpo/q_t": 0.4129534363746643, + "grad_norm": 21.814550399780273, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.3751649856567383, + "logits/rejected": -0.36549320816993713, + "logps/chosen": -249.61468505859375, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -420.3258056640625, + "loss": 1.1125, + "margin_dpo/margin_mean": 137.64418029785156, + "margin_dpo/margin_std": 192.5943145751953, + "step": 255 + }, + { + "KL/chosen_KL_mean": -163.554931640625, + "KL/mean": -237.36813354492188, + "KL/rejected_KL_mean": -311.1813049316406, + "KL/std": 160.40505981445312, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.002771299798041582, + "fcm_dpo/delta": -0.010388858616352081, + "fcm_dpo/margin": 147.62640380859375, + "fcm_dpo/q_t": 0.4049738049507141, + "grad_norm": 22.84318733215332, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.2948145270347595, + "logits/rejected": -0.30300652980804443, + "logps/chosen": -218.71841430664062, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -403.7442321777344, + "loss": 1.0791, + "margin_dpo/margin_mean": 147.62640380859375, + "margin_dpo/margin_std": 183.9723663330078, + "step": 256 + }, + { + "KL/chosen_KL_mean": -159.0382080078125, + "KL/mean": -236.62881469726562, + "KL/rejected_KL_mean": -314.2193603515625, + "KL/std": 167.221923828125, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.0026884416583925486, + "fcm_dpo/delta": -0.022461801767349243, + "fcm_dpo/margin": 155.18118286132812, + "fcm_dpo/q_t": 0.4032364785671234, + "grad_norm": 20.142290115356445, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.3664169907569885, + "logits/rejected": -0.3535708785057068, + "logps/chosen": -208.4619140625, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -393.7572937011719, + "loss": 1.0773, + "margin_dpo/margin_mean": 155.18118286132812, + "margin_dpo/margin_std": 189.4936065673828, + "step": 257 + }, + { + "KL/chosen_KL_mean": -215.01577758789062, + "KL/mean": -290.2253112792969, + "KL/rejected_KL_mean": -365.4348449707031, + "KL/std": 179.46685791015625, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.002726411446928978, + "fcm_dpo/delta": -0.01161413174122572, + "fcm_dpo/margin": 150.41905212402344, + "fcm_dpo/q_t": 0.4042818546295166, + "grad_norm": 21.353389739990234, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.34295833110809326, + "logits/rejected": -0.3423942029476166, + "logps/chosen": -274.39990234375, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -461.4249267578125, + "loss": 1.0973, + "margin_dpo/margin_mean": 150.41903686523438, + "margin_dpo/margin_std": 214.1118621826172, + "step": 258 + }, + { + "KL/chosen_KL_mean": -200.76553344726562, + "KL/mean": -263.8870544433594, + "KL/rejected_KL_mean": -327.0085754394531, + "KL/std": 171.11920166015625, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.002754590008407831, + "fcm_dpo/delta": 0.05827288702130318, + "fcm_dpo/margin": 126.2430419921875, + "fcm_dpo/q_t": 0.42151179909706116, + "grad_norm": 21.743154525756836, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.38276469707489014, + "logits/rejected": -0.38169676065444946, + "logps/chosen": -253.59388732910156, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -416.2002258300781, + "loss": 1.1374, + "margin_dpo/margin_mean": 126.2430419921875, + "margin_dpo/margin_std": 194.41262817382812, + "step": 259 + }, + { + "KL/chosen_KL_mean": -205.74594116210938, + "KL/mean": -292.67694091796875, + "KL/rejected_KL_mean": -379.60797119140625, + "KL/std": 180.63450622558594, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.0027318743523210287, + "fcm_dpo/delta": -0.08736619353294373, + "fcm_dpo/margin": 173.8620147705078, + "fcm_dpo/q_t": 0.3907129168510437, + "grad_norm": 23.710599899291992, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.3454121947288513, + "logits/rejected": -0.3613763451576233, + "logps/chosen": -253.16363525390625, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -474.69775390625, + "loss": 1.0298, + "margin_dpo/margin_mean": 173.86199951171875, + "margin_dpo/margin_std": 194.74197387695312, + "step": 260 + }, + { + "KL/chosen_KL_mean": -212.67572021484375, + "KL/mean": -292.53973388671875, + "KL/rejected_KL_mean": -372.40374755859375, + "KL/std": 194.0233154296875, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0026178224943578243, + "fcm_dpo/delta": -0.02129589021205902, + "fcm_dpo/margin": 159.72799682617188, + "fcm_dpo/q_t": 0.4042361378669739, + "grad_norm": 20.83995819091797, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.2934446334838867, + "logits/rejected": -0.2951222062110901, + "logps/chosen": -265.70709228515625, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -460.918701171875, + "loss": 1.0846, + "margin_dpo/margin_mean": 159.72799682617188, + "margin_dpo/margin_std": 214.90869140625, + "step": 261 + }, + { + "KL/chosen_KL_mean": -250.020751953125, + "KL/mean": -315.95758056640625, + "KL/rejected_KL_mean": -381.8944091796875, + "KL/std": 176.18783569335938, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.002712658606469631, + "fcm_dpo/delta": 0.04593823477625847, + "fcm_dpo/margin": 131.87368774414062, + "fcm_dpo/q_t": 0.41718244552612305, + "grad_norm": 25.73346519470215, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.2960895001888275, + "logits/rejected": -0.2673763632774353, + "logps/chosen": -309.6408996582031, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -468.3129577636719, + "loss": 1.1128, + "margin_dpo/margin_mean": 131.87368774414062, + "margin_dpo/margin_std": 173.75973510742188, + "step": 262 + }, + { + "KL/chosen_KL_mean": -227.78350830078125, + "KL/mean": -321.23602294921875, + "KL/rejected_KL_mean": -414.6885070800781, + "KL/std": 205.6634521484375, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.0025594395119696856, + "fcm_dpo/delta": -0.09393350034952164, + "fcm_dpo/margin": 186.90499877929688, + "fcm_dpo/q_t": 0.38841354846954346, + "grad_norm": 20.733972549438477, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.34038394689559937, + "logits/rejected": -0.31836265325546265, + "logps/chosen": -287.2044677734375, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -511.54571533203125, + "loss": 1.0375, + "margin_dpo/margin_mean": 186.90501403808594, + "margin_dpo/margin_std": 219.97149658203125, + "step": 263 + }, + { + "KL/chosen_KL_mean": -241.496826171875, + "KL/mean": -321.6051025390625, + "KL/rejected_KL_mean": -401.7133483886719, + "KL/std": 193.909912109375, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.002497072098776698, + "fcm_dpo/delta": -0.003351174294948578, + "fcm_dpo/margin": 160.21653747558594, + "fcm_dpo/q_t": 0.4085013270378113, + "grad_norm": 24.449872970581055, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.38598304986953735, + "logits/rejected": -0.3787904977798462, + "logps/chosen": -304.2189025878906, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -495.569580078125, + "loss": 1.1026, + "margin_dpo/margin_mean": 160.21652221679688, + "margin_dpo/margin_std": 222.1091766357422, + "step": 264 + }, + { + "KL/chosen_KL_mean": -263.1861877441406, + "KL/mean": -338.3819274902344, + "KL/rejected_KL_mean": -413.5776672363281, + "KL/std": 215.61642456054688, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.0025649338494986296, + "fcm_dpo/delta": 0.016129544004797935, + "fcm_dpo/margin": 150.3914794921875, + "fcm_dpo/q_t": 0.4140586256980896, + "grad_norm": 27.338926315307617, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.315537691116333, + "logits/rejected": -0.2989951968193054, + "logps/chosen": -325.15765380859375, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -501.5982666015625, + "loss": 1.1354, + "margin_dpo/margin_mean": 150.3914794921875, + "margin_dpo/margin_std": 256.43670654296875, + "step": 265 + }, + { + "KL/chosen_KL_mean": -267.7913513183594, + "KL/mean": -329.1666259765625, + "KL/rejected_KL_mean": -390.54193115234375, + "KL/std": 179.03036499023438, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0026689646765589714, + "fcm_dpo/delta": 0.0801931619644165, + "fcm_dpo/margin": 122.75053405761719, + "fcm_dpo/q_t": 0.42658764123916626, + "grad_norm": 51.205745697021484, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.4160683751106262, + "logits/rejected": -0.38314324617385864, + "logps/chosen": -334.8910217285156, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -458.51312255859375, + "loss": 1.1579, + "margin_dpo/margin_mean": 122.75054931640625, + "margin_dpo/margin_std": 208.8185577392578, + "step": 266 + }, + { + "KL/chosen_KL_mean": -235.4722900390625, + "KL/mean": -310.92205810546875, + "KL/rejected_KL_mean": -386.3717956542969, + "KL/std": 192.781982421875, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.0026693195104599, + "fcm_dpo/delta": -0.003913886845111847, + "fcm_dpo/margin": 150.89947509765625, + "fcm_dpo/q_t": 0.4085218012332916, + "grad_norm": 33.3153190612793, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.40000247955322266, + "logits/rejected": -0.3701097071170807, + "logps/chosen": -304.44305419921875, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -476.54022216796875, + "loss": 1.1014, + "margin_dpo/margin_mean": 150.89947509765625, + "margin_dpo/margin_std": 218.03158569335938, + "step": 267 + }, + { + "KL/chosen_KL_mean": -245.03106689453125, + "KL/mean": -320.18157958984375, + "KL/rejected_KL_mean": -395.33209228515625, + "KL/std": 184.99560546875, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0026660324074327946, + "fcm_dpo/delta": -0.0015031322836875916, + "fcm_dpo/margin": 150.30104064941406, + "fcm_dpo/q_t": 0.4092382788658142, + "grad_norm": 35.57260513305664, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.3661958575248718, + "logits/rejected": -0.37029725313186646, + "logps/chosen": -300.931396484375, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -496.979736328125, + "loss": 1.1057, + "margin_dpo/margin_mean": 150.30104064941406, + "margin_dpo/margin_std": 222.54318237304688, + "step": 268 + }, + { + "KL/chosen_KL_mean": -247.552490234375, + "KL/mean": -334.0849609375, + "KL/rejected_KL_mean": -420.617431640625, + "KL/std": 184.9167022705078, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.002623080974444747, + "fcm_dpo/delta": -0.06239096447825432, + "fcm_dpo/margin": 173.06497192382812, + "fcm_dpo/q_t": 0.3951506018638611, + "grad_norm": 33.0728645324707, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.4484559893608093, + "logits/rejected": -0.43009868264198303, + "logps/chosen": -317.592041015625, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -527.966796875, + "loss": 1.0688, + "margin_dpo/margin_mean": 173.06497192382812, + "margin_dpo/margin_std": 231.93914794921875, + "step": 269 + }, + { + "KL/chosen_KL_mean": -216.1859130859375, + "KL/mean": -281.20635986328125, + "KL/rejected_KL_mean": -346.226806640625, + "KL/std": 163.6240234375, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.002685854909941554, + "fcm_dpo/delta": 0.055218860507011414, + "fcm_dpo/margin": 130.04087829589844, + "fcm_dpo/q_t": 0.4206189513206482, + "grad_norm": 33.42424774169922, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.48196592926979065, + "logits/rejected": -0.47910600900650024, + "logps/chosen": -285.7193908691406, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -456.15545654296875, + "loss": 1.1401, + "margin_dpo/margin_mean": 130.04086303710938, + "margin_dpo/margin_std": 206.94802856445312, + "step": 270 + }, + { + "KL/chosen_KL_mean": -207.62020874023438, + "KL/mean": -292.3222351074219, + "KL/rejected_KL_mean": -377.0242614746094, + "KL/std": 170.31471252441406, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.0025946078822016716, + "fcm_dpo/delta": -0.047072481364011765, + "fcm_dpo/margin": 169.404052734375, + "fcm_dpo/q_t": 0.3980659544467926, + "grad_norm": 26.481239318847656, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.3720319867134094, + "logits/rejected": -0.35801082849502563, + "logps/chosen": -264.384765625, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -469.5380859375, + "loss": 1.0544, + "margin_dpo/margin_mean": 169.404052734375, + "margin_dpo/margin_std": 195.69192504882812, + "step": 271 + }, + { + "KL/chosen_KL_mean": -196.43861389160156, + "KL/mean": -296.65728759765625, + "KL/rejected_KL_mean": -396.8759460449219, + "KL/std": 188.86888122558594, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.0024972385726869106, + "fcm_dpo/delta": -0.11782557517290115, + "fcm_dpo/margin": 200.43734741210938, + "fcm_dpo/q_t": 0.3803797960281372, + "grad_norm": 34.67290115356445, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.3195474445819855, + "logits/rejected": -0.3320087790489197, + "logps/chosen": -245.93577575683594, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -502.41876220703125, + "loss": 1.0039, + "margin_dpo/margin_mean": 200.43734741210938, + "margin_dpo/margin_std": 193.63595581054688, + "step": 272 + }, + { + "KL/chosen_KL_mean": -231.34429931640625, + "KL/mean": -329.0470275878906, + "KL/rejected_KL_mean": -426.7497863769531, + "KL/std": 190.58169555664062, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.002335872035473585, + "fcm_dpo/delta": -0.06662485003471375, + "fcm_dpo/margin": 195.40545654296875, + "fcm_dpo/q_t": 0.3928803503513336, + "grad_norm": 23.508026123046875, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.34439030289649963, + "logits/rejected": -0.3275430202484131, + "logps/chosen": -294.3197021484375, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -519.2483520507812, + "loss": 1.0439, + "margin_dpo/margin_mean": 195.40545654296875, + "margin_dpo/margin_std": 221.81558227539062, + "step": 273 + }, + { + "KL/chosen_KL_mean": -271.6736755371094, + "KL/mean": -345.9742431640625, + "KL/rejected_KL_mean": -420.2747802734375, + "KL/std": 178.9398193359375, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.002406273502856493, + "fcm_dpo/delta": 0.04620227962732315, + "fcm_dpo/margin": 148.60110473632812, + "fcm_dpo/q_t": 0.41794371604919434, + "grad_norm": 29.284175872802734, + "learning_rate": 3.75e-07, + "logits/chosen": -0.26377707719802856, + "logits/rejected": -0.24428263306617737, + "logps/chosen": -327.34136962890625, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -497.60784912109375, + "loss": 1.12, + "margin_dpo/margin_mean": 148.60110473632812, + "margin_dpo/margin_std": 207.24087524414062, + "step": 274 + }, + { + "KL/chosen_KL_mean": -218.38369750976562, + "KL/mean": -301.8189697265625, + "KL/rejected_KL_mean": -385.2542419433594, + "KL/std": 182.27256774902344, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.002414708025753498, + "fcm_dpo/delta": -0.0038442090153694153, + "fcm_dpo/margin": 166.87054443359375, + "fcm_dpo/q_t": 0.40621882677078247, + "grad_norm": 23.91200828552246, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.29596638679504395, + "logits/rejected": -0.30184778571128845, + "logps/chosen": -266.9783935546875, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -478.55792236328125, + "loss": 1.082, + "margin_dpo/margin_mean": 166.87054443359375, + "margin_dpo/margin_std": 204.91339111328125, + "step": 275 + }, + { + "KL/chosen_KL_mean": -243.04812622070312, + "KL/mean": -326.12738037109375, + "KL/rejected_KL_mean": -409.20660400390625, + "KL/std": 193.84124755859375, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.0023798246402293444, + "fcm_dpo/delta": 0.004829345270991325, + "fcm_dpo/margin": 166.15847778320312, + "fcm_dpo/q_t": 0.41078272461891174, + "grad_norm": 28.621992111206055, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.27831506729125977, + "logits/rejected": -0.25273844599723816, + "logps/chosen": -299.62554931640625, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -479.572265625, + "loss": 1.1044, + "margin_dpo/margin_mean": 166.15847778320312, + "margin_dpo/margin_std": 238.60415649414062, + "step": 276 + }, + { + "KL/chosen_KL_mean": -268.6829833984375, + "KL/mean": -354.7122802734375, + "KL/rejected_KL_mean": -440.74163818359375, + "KL/std": 195.40762329101562, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.002399809192866087, + "fcm_dpo/delta": -0.014955121092498302, + "fcm_dpo/margin": 172.05862426757812, + "fcm_dpo/q_t": 0.4049604535102844, + "grad_norm": 21.864580154418945, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.3014830946922302, + "logits/rejected": -0.28200146555900574, + "logps/chosen": -324.95452880859375, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -533.6229248046875, + "loss": 1.0895, + "margin_dpo/margin_mean": 172.05862426757812, + "margin_dpo/margin_std": 233.0054931640625, + "step": 277 + }, + { + "KL/chosen_KL_mean": -236.03111267089844, + "KL/mean": -337.50164794921875, + "KL/rejected_KL_mean": -438.97216796875, + "KL/std": 204.71511840820312, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.0022814148105680943, + "fcm_dpo/delta": -0.0743660181760788, + "fcm_dpo/margin": 202.94107055664062, + "fcm_dpo/q_t": 0.39208030700683594, + "grad_norm": 22.533647537231445, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.3116120994091034, + "logits/rejected": -0.3181500732898712, + "logps/chosen": -288.9730529785156, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -530.2257690429688, + "loss": 1.0391, + "margin_dpo/margin_mean": 202.94107055664062, + "margin_dpo/margin_std": 229.4819793701172, + "step": 278 + }, + { + "KL/chosen_KL_mean": -269.08013916015625, + "KL/mean": -369.6549377441406, + "KL/rejected_KL_mean": -470.229736328125, + "KL/std": 221.88092041015625, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.002186977071687579, + "fcm_dpo/delta": -0.049054618924856186, + "fcm_dpo/margin": 201.14959716796875, + "fcm_dpo/q_t": 0.39946407079696655, + "grad_norm": 28.020355224609375, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.26989489793777466, + "logits/rejected": -0.27033862471580505, + "logps/chosen": -317.7214660644531, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -558.0811767578125, + "loss": 1.077, + "margin_dpo/margin_mean": 201.14959716796875, + "margin_dpo/margin_std": 272.2268371582031, + "step": 279 + }, + { + "KL/chosen_KL_mean": -250.334228515625, + "KL/mean": -348.6322021484375, + "KL/rejected_KL_mean": -446.93011474609375, + "KL/std": 190.50103759765625, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.002175545319914818, + "fcm_dpo/delta": -0.03181631863117218, + "fcm_dpo/margin": 196.59591674804688, + "fcm_dpo/q_t": 0.3984387516975403, + "grad_norm": 23.84842300415039, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.34399133920669556, + "logits/rejected": -0.34695863723754883, + "logps/chosen": -309.13134765625, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -545.5489501953125, + "loss": 1.0504, + "margin_dpo/margin_mean": 196.59591674804688, + "margin_dpo/margin_std": 204.2347412109375, + "step": 280 + }, + { + "KL/chosen_KL_mean": -232.19122314453125, + "KL/mean": -317.8921813964844, + "KL/rejected_KL_mean": -403.5931701660156, + "KL/std": 194.03050231933594, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.002210037549957633, + "fcm_dpo/delta": 0.023013081401586533, + "fcm_dpo/margin": 171.40191650390625, + "fcm_dpo/q_t": 0.41006746888160706, + "grad_norm": 25.59185218811035, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.36414363980293274, + "logits/rejected": -0.3377975821495056, + "logps/chosen": -287.67974853515625, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -484.4757385253906, + "loss": 1.0946, + "margin_dpo/margin_mean": 171.40191650390625, + "margin_dpo/margin_std": 207.35641479492188, + "step": 281 + }, + { + "KL/chosen_KL_mean": -260.83099365234375, + "KL/mean": -327.07366943359375, + "KL/rejected_KL_mean": -393.31640625, + "KL/std": 194.19290161132812, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.0022110575810074806, + "fcm_dpo/delta": 0.021844828501343727, + "fcm_dpo/margin": 132.48541259765625, + "fcm_dpo/q_t": 0.4335850477218628, + "grad_norm": 26.0980224609375, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.4269988536834717, + "logits/rejected": -0.40729379653930664, + "logps/chosen": -333.901123046875, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -488.6673889160156, + "loss": 1.1691, + "margin_dpo/margin_mean": 132.48541259765625, + "margin_dpo/margin_std": 221.13290405273438, + "step": 282 + }, + { + "KL/chosen_KL_mean": -268.7425537109375, + "KL/mean": -372.56036376953125, + "KL/rejected_KL_mean": -476.37823486328125, + "KL/std": 239.3566131591797, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.0021882583387196064, + "fcm_dpo/delta": -0.06307591497898102, + "fcm_dpo/margin": 207.63565063476562, + "fcm_dpo/q_t": 0.3927934765815735, + "grad_norm": 42.36301803588867, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.3655827045440674, + "logits/rejected": -0.3707088232040405, + "logps/chosen": -330.6409912109375, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -573.36474609375, + "loss": 1.0442, + "margin_dpo/margin_mean": 207.63565063476562, + "margin_dpo/margin_std": 228.7406005859375, + "step": 283 + }, + { + "KL/chosen_KL_mean": -257.25982666015625, + "KL/mean": -356.34381103515625, + "KL/rejected_KL_mean": -455.427734375, + "KL/std": 220.77102661132812, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0021124526392668486, + "fcm_dpo/delta": -0.021564599126577377, + "fcm_dpo/margin": 198.16790771484375, + "fcm_dpo/q_t": 0.40301263332366943, + "grad_norm": 27.41075325012207, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.39809176325798035, + "logits/rejected": -0.3830932378768921, + "logps/chosen": -315.69537353515625, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -548.89697265625, + "loss": 1.0657, + "margin_dpo/margin_mean": 198.16790771484375, + "margin_dpo/margin_std": 231.41705322265625, + "step": 284 + }, + { + "KL/chosen_KL_mean": -294.6321716308594, + "KL/mean": -379.30816650390625, + "KL/rejected_KL_mean": -463.9841613769531, + "KL/std": 188.61007690429688, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.0021794699132442474, + "fcm_dpo/delta": 0.03255731239914894, + "fcm_dpo/margin": 169.35198974609375, + "fcm_dpo/q_t": 0.4124819040298462, + "grad_norm": 22.987781524658203, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.38745754957199097, + "logits/rejected": -0.37279653549194336, + "logps/chosen": -360.8643798828125, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -563.1109619140625, + "loss": 1.1036, + "margin_dpo/margin_mean": 169.35198974609375, + "margin_dpo/margin_std": 209.543212890625, + "step": 285 + }, + { + "KL/chosen_KL_mean": -294.52862548828125, + "KL/mean": -389.8953857421875, + "KL/rejected_KL_mean": -485.26220703125, + "KL/std": 226.58419799804688, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.002156574744731188, + "fcm_dpo/delta": -0.01306964922696352, + "fcm_dpo/margin": 190.7335662841797, + "fcm_dpo/q_t": 0.4043489992618561, + "grad_norm": 29.59273910522461, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.39581769704818726, + "logits/rejected": -0.3823132812976837, + "logps/chosen": -367.4796142578125, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -573.8506469726562, + "loss": 1.0835, + "margin_dpo/margin_mean": 190.73355102539062, + "margin_dpo/margin_std": 245.71502685546875, + "step": 286 + }, + { + "KL/chosen_KL_mean": -271.3664855957031, + "KL/mean": -352.49114990234375, + "KL/rejected_KL_mean": -433.61578369140625, + "KL/std": 198.40789794921875, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.002192837418988347, + "fcm_dpo/delta": 0.04933007434010506, + "fcm_dpo/margin": 162.24928283691406, + "fcm_dpo/q_t": 0.41866976022720337, + "grad_norm": 22.294658660888672, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.335421085357666, + "logits/rejected": -0.3113495111465454, + "logps/chosen": -332.90765380859375, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -511.31182861328125, + "loss": 1.1168, + "margin_dpo/margin_mean": 162.24928283691406, + "margin_dpo/margin_std": 221.52276611328125, + "step": 287 + }, + { + "KL/chosen_KL_mean": -259.2959289550781, + "KL/mean": -359.46661376953125, + "KL/rejected_KL_mean": -459.6372985839844, + "KL/std": 196.78070068359375, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.002125432016327977, + "fcm_dpo/delta": -0.03373609483242035, + "fcm_dpo/margin": 200.34136962890625, + "fcm_dpo/q_t": 0.39845049381256104, + "grad_norm": 19.78911018371582, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.32467448711395264, + "logits/rejected": -0.3095056116580963, + "logps/chosen": -315.9571533203125, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -546.9730224609375, + "loss": 1.0563, + "margin_dpo/margin_mean": 200.34136962890625, + "margin_dpo/margin_std": 207.9046630859375, + "step": 288 + }, + { + "KL/chosen_KL_mean": -257.16033935546875, + "KL/mean": -367.7012939453125, + "KL/rejected_KL_mean": -478.2422180175781, + "KL/std": 222.08558654785156, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.0021074905525892973, + "fcm_dpo/delta": -0.07651316374540329, + "fcm_dpo/margin": 221.08187866210938, + "fcm_dpo/q_t": 0.39082980155944824, + "grad_norm": 25.83208465576172, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.2596939206123352, + "logits/rejected": -0.2600315511226654, + "logps/chosen": -302.3907470703125, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -565.8848876953125, + "loss": 1.0343, + "margin_dpo/margin_mean": 221.08187866210938, + "margin_dpo/margin_std": 239.01174926757812, + "step": 289 + }, + { + "KL/chosen_KL_mean": -288.99212646484375, + "KL/mean": -400.0706787109375, + "KL/rejected_KL_mean": -511.1492919921875, + "KL/std": 235.73135375976562, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.0020047822035849094, + "fcm_dpo/delta": -0.05289486423134804, + "fcm_dpo/margin": 222.15716552734375, + "fcm_dpo/q_t": 0.39741161465644836, + "grad_norm": 21.596309661865234, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.2785472571849823, + "logits/rejected": -0.30120235681533813, + "logps/chosen": -344.463623046875, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -627.8578491210938, + "loss": 1.0601, + "margin_dpo/margin_mean": 222.15716552734375, + "margin_dpo/margin_std": 277.6689758300781, + "step": 290 + }, + { + "KL/chosen_KL_mean": -259.6630859375, + "KL/mean": -363.2607421875, + "KL/rejected_KL_mean": -466.8583984375, + "KL/std": 208.01739501953125, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.002006969880312681, + "fcm_dpo/delta": -0.018925651907920837, + "fcm_dpo/margin": 207.1953125, + "fcm_dpo/q_t": 0.4023928642272949, + "grad_norm": 20.128128051757812, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3157449960708618, + "logits/rejected": -0.3077038526535034, + "logps/chosen": -322.9234619140625, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -556.155517578125, + "loss": 1.0734, + "margin_dpo/margin_mean": 207.1953125, + "margin_dpo/margin_std": 247.18600463867188, + "step": 291 + }, + { + "KL/chosen_KL_mean": -291.982666015625, + "KL/mean": -398.6346740722656, + "KL/rejected_KL_mean": -505.2866516113281, + "KL/std": 242.49713134765625, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.00197390653192997, + "fcm_dpo/delta": -0.024488601833581924, + "fcm_dpo/margin": 213.30393981933594, + "fcm_dpo/q_t": 0.40195518732070923, + "grad_norm": 20.321792602539062, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.26276931166648865, + "logits/rejected": -0.24905674159526825, + "logps/chosen": -345.9012145996094, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -595.248046875, + "loss": 1.0846, + "margin_dpo/margin_mean": 213.303955078125, + "margin_dpo/margin_std": 287.27978515625, + "step": 292 + }, + { + "KL/chosen_KL_mean": -286.88641357421875, + "KL/mean": -371.2066650390625, + "KL/rejected_KL_mean": -455.52691650390625, + "KL/std": 238.20947265625, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.002014589263126254, + "fcm_dpo/delta": 0.06667286157608032, + "fcm_dpo/margin": 168.64053344726562, + "fcm_dpo/q_t": 0.42376092076301575, + "grad_norm": 30.699453353881836, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.30312398076057434, + "logits/rejected": -0.2980945110321045, + "logps/chosen": -347.262451171875, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -533.37939453125, + "loss": 1.1428, + "margin_dpo/margin_mean": 168.64053344726562, + "margin_dpo/margin_std": 268.50054931640625, + "step": 293 + }, + { + "KL/chosen_KL_mean": -258.9250793457031, + "KL/mean": -354.4201354980469, + "KL/rejected_KL_mean": -449.91522216796875, + "KL/std": 209.728759765625, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.002027429174631834, + "fcm_dpo/delta": 0.014280688017606735, + "fcm_dpo/margin": 190.99014282226562, + "fcm_dpo/q_t": 0.41201189160346985, + "grad_norm": 20.131216049194336, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.31998908519744873, + "logits/rejected": -0.32771599292755127, + "logps/chosen": -307.0126037597656, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -531.8121948242188, + "loss": 1.1023, + "margin_dpo/margin_mean": 190.99014282226562, + "margin_dpo/margin_std": 262.94915771484375, + "step": 294 + }, + { + "KL/chosen_KL_mean": -308.3461608886719, + "KL/mean": -412.640869140625, + "KL/rejected_KL_mean": -516.9356079101562, + "KL/std": 238.06198120117188, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.0020288482774049044, + "fcm_dpo/delta": -0.026576882228255272, + "fcm_dpo/margin": 208.58944702148438, + "fcm_dpo/q_t": 0.403533399105072, + "grad_norm": 23.50198745727539, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.3032621741294861, + "logits/rejected": -0.30784791707992554, + "logps/chosen": -358.27081298828125, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -604.3919067382812, + "loss": 1.0866, + "margin_dpo/margin_mean": 208.5894317626953, + "margin_dpo/margin_std": 286.6007385253906, + "step": 295 + }, + { + "KL/chosen_KL_mean": -373.33184814453125, + "KL/mean": -444.53790283203125, + "KL/rejected_KL_mean": -515.743896484375, + "KL/std": 225.14468383789062, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.0020105030853301287, + "fcm_dpo/delta": -0.001359270652756095, + "fcm_dpo/margin": 142.41207885742188, + "fcm_dpo/q_t": 0.4332321584224701, + "grad_norm": 26.687498092651367, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.3519822359085083, + "logits/rejected": -0.3364889621734619, + "logps/chosen": -438.82305908203125, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -608.8330078125, + "loss": 1.1998, + "margin_dpo/margin_mean": 142.41207885742188, + "margin_dpo/margin_std": 283.15576171875, + "step": 296 + }, + { + "KL/chosen_KL_mean": -318.62396240234375, + "KL/mean": -401.7255859375, + "KL/rejected_KL_mean": -484.8272399902344, + "KL/std": 208.9388427734375, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.0020615635439753532, + "fcm_dpo/delta": 0.06406788527965546, + "fcm_dpo/margin": 166.20327758789062, + "fcm_dpo/q_t": 0.42071211338043213, + "grad_norm": 33.74131393432617, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.3442198634147644, + "logits/rejected": -0.3471313714981079, + "logps/chosen": -375.10089111328125, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -579.9658203125, + "loss": 1.1181, + "margin_dpo/margin_mean": 166.20327758789062, + "margin_dpo/margin_std": 211.13070678710938, + "step": 297 + }, + { + "KL/chosen_KL_mean": -332.2897033691406, + "KL/mean": -439.97723388671875, + "KL/rejected_KL_mean": -547.6647338867188, + "KL/std": 272.48065185546875, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.0020646718330681324, + "fcm_dpo/delta": -0.05182594805955887, + "fcm_dpo/margin": 215.37503051757812, + "fcm_dpo/q_t": 0.40108564496040344, + "grad_norm": 25.438440322875977, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.40182167291641235, + "logits/rejected": -0.4196791350841522, + "logps/chosen": -399.6148681640625, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -664.326904296875, + "loss": 1.0866, + "margin_dpo/margin_mean": 215.37503051757812, + "margin_dpo/margin_std": 319.04156494140625, + "step": 298 + }, + { + "KL/chosen_KL_mean": -269.0082092285156, + "KL/mean": -362.99334716796875, + "KL/rejected_KL_mean": -456.9785461425781, + "KL/std": 204.53512573242188, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.0020705664064735174, + "fcm_dpo/delta": 0.009840298444032669, + "fcm_dpo/margin": 187.97030639648438, + "fcm_dpo/q_t": 0.4111258089542389, + "grad_norm": 25.296497344970703, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.3781265616416931, + "logits/rejected": -0.38146746158599854, + "logps/chosen": -317.9703063964844, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -541.3067626953125, + "loss": 1.1022, + "margin_dpo/margin_mean": 187.97030639648438, + "margin_dpo/margin_std": 254.50332641601562, + "step": 299 + }, + { + "KL/chosen_KL_mean": -330.77734375, + "KL/mean": -445.33099365234375, + "KL/rejected_KL_mean": -559.8846435546875, + "KL/std": 249.59640502929688, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0019939406774938107, + "fcm_dpo/delta": -0.06574904173612595, + "fcm_dpo/margin": 229.10726928710938, + "fcm_dpo/q_t": 0.3942931890487671, + "grad_norm": 23.738195419311523, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.3730261027812958, + "logits/rejected": -0.3780125379562378, + "logps/chosen": -389.8510437011719, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -655.85107421875, + "loss": 1.0677, + "margin_dpo/margin_mean": 229.10726928710938, + "margin_dpo/margin_std": 303.4993896484375, + "step": 300 + }, + { + "KL/chosen_KL_mean": -281.2186584472656, + "KL/mean": -389.272705078125, + "KL/rejected_KL_mean": -497.3267517089844, + "KL/std": 221.46356201171875, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.0019263834692537785, + "fcm_dpo/delta": -0.019256018102169037, + "fcm_dpo/margin": 216.10809326171875, + "fcm_dpo/q_t": 0.4040316045284271, + "grad_norm": 22.376789093017578, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.41540956497192383, + "logits/rejected": -0.417570024728775, + "logps/chosen": -338.468017578125, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -589.6802978515625, + "loss": 1.0727, + "margin_dpo/margin_mean": 216.10809326171875, + "margin_dpo/margin_std": 264.18560791015625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -252.17800903320312, + "KL/mean": -345.2618103027344, + "KL/rejected_KL_mean": -438.34564208984375, + "KL/std": 204.30517578125, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.001948266290128231, + "fcm_dpo/delta": 0.04131300002336502, + "fcm_dpo/margin": 186.16763305664062, + "fcm_dpo/q_t": 0.41672298312187195, + "grad_norm": 20.922119140625, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.3827202320098877, + "logits/rejected": -0.3872525095939636, + "logps/chosen": -303.3759765625, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -535.572021484375, + "loss": 1.1105, + "margin_dpo/margin_mean": 186.16763305664062, + "margin_dpo/margin_std": 242.23690795898438, + "step": 302 + }, + { + "KL/chosen_KL_mean": -265.6185302734375, + "KL/mean": -350.90936279296875, + "KL/rejected_KL_mean": -436.20013427734375, + "KL/std": 210.7963104248047, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002027076669037342, + "fcm_dpo/delta": 0.06060871109366417, + "fcm_dpo/margin": 170.58163452148438, + "fcm_dpo/q_t": 0.42100226879119873, + "grad_norm": 23.99404525756836, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.39151397347450256, + "logits/rejected": -0.37832212448120117, + "logps/chosen": -332.33245849609375, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -523.1455688476562, + "loss": 1.1324, + "margin_dpo/margin_mean": 170.58163452148438, + "margin_dpo/margin_std": 250.7506103515625, + "step": 303 + }, + { + "KL/chosen_KL_mean": -237.57508850097656, + "KL/mean": -335.66131591796875, + "KL/rejected_KL_mean": -433.74755859375, + "KL/std": 193.1407470703125, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.0020741671323776245, + "fcm_dpo/delta": -0.008699264377355576, + "fcm_dpo/margin": 196.17247009277344, + "fcm_dpo/q_t": 0.40223217010498047, + "grad_norm": 29.905309677124023, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.3983379900455475, + "logits/rejected": -0.39082038402557373, + "logps/chosen": -309.5257873535156, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -524.2196044921875, + "loss": 1.0511, + "margin_dpo/margin_mean": 196.1724853515625, + "margin_dpo/margin_std": 167.17166137695312, + "step": 304 + }, + { + "KL/chosen_KL_mean": -263.044921875, + "KL/mean": -349.27886962890625, + "KL/rejected_KL_mean": -435.51275634765625, + "KL/std": 222.33639526367188, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.002054195385426283, + "fcm_dpo/delta": 0.049560483545064926, + "fcm_dpo/margin": 172.4678497314453, + "fcm_dpo/q_t": 0.4197576642036438, + "grad_norm": 21.999088287353516, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.4084005057811737, + "logits/rejected": -0.39455413818359375, + "logps/chosen": -329.84014892578125, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -528.267333984375, + "loss": 1.1263, + "margin_dpo/margin_mean": 172.46783447265625, + "margin_dpo/margin_std": 243.25338745117188, + "step": 305 + }, + { + "KL/chosen_KL_mean": -266.616943359375, + "KL/mean": -349.9216613769531, + "KL/rejected_KL_mean": -433.22637939453125, + "KL/std": 198.0668182373047, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.0021604858338832855, + "fcm_dpo/delta": 0.04456178471446037, + "fcm_dpo/margin": 166.60940551757812, + "fcm_dpo/q_t": 0.41712498664855957, + "grad_norm": 26.718704223632812, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.3759077787399292, + "logits/rejected": -0.3533060550689697, + "logps/chosen": -336.30084228515625, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -518.3855590820312, + "loss": 1.1077, + "margin_dpo/margin_mean": 166.60940551757812, + "margin_dpo/margin_std": 213.0517578125, + "step": 306 + }, + { + "KL/chosen_KL_mean": -253.76490783691406, + "KL/mean": -340.68011474609375, + "KL/rejected_KL_mean": -427.5953369140625, + "KL/std": 189.57736206054688, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.002179474337026477, + "fcm_dpo/delta": 0.023815887048840523, + "fcm_dpo/margin": 173.8303985595703, + "fcm_dpo/q_t": 0.41250067949295044, + "grad_norm": 29.69277000427246, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.3497428596019745, + "logits/rejected": -0.33149683475494385, + "logps/chosen": -323.9303283691406, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -514.567626953125, + "loss": 1.103, + "margin_dpo/margin_mean": 173.83041381835938, + "margin_dpo/margin_std": 232.69435119628906, + "step": 307 + }, + { + "KL/chosen_KL_mean": -262.7245178222656, + "KL/mean": -362.1474609375, + "KL/rejected_KL_mean": -461.57037353515625, + "KL/std": 215.58865356445312, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.002166676102206111, + "fcm_dpo/delta": -0.035376034677028656, + "fcm_dpo/margin": 198.84585571289062, + "fcm_dpo/q_t": 0.3978816568851471, + "grad_norm": 28.970443725585938, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.3525085151195526, + "logits/rejected": -0.34443122148513794, + "logps/chosen": -317.9695129394531, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -540.942626953125, + "loss": 1.0487, + "margin_dpo/margin_mean": 198.84584045410156, + "margin_dpo/margin_std": 205.41067504882812, + "step": 308 + }, + { + "KL/chosen_KL_mean": -271.55572509765625, + "KL/mean": -371.1033935546875, + "KL/rejected_KL_mean": -470.651123046875, + "KL/std": 229.63232421875, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0021552760154008865, + "fcm_dpo/delta": -0.034857943654060364, + "fcm_dpo/margin": 199.0953826904297, + "fcm_dpo/q_t": 0.39886415004730225, + "grad_norm": 31.433834075927734, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.32273930311203003, + "logits/rejected": -0.319173663854599, + "logps/chosen": -320.5148010253906, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -552.9918212890625, + "loss": 1.0704, + "margin_dpo/margin_mean": 199.09536743164062, + "margin_dpo/margin_std": 241.08505249023438, + "step": 309 + }, + { + "KL/chosen_KL_mean": -317.44281005859375, + "KL/mean": -406.10400390625, + "KL/rejected_KL_mean": -494.7651672363281, + "KL/std": 207.94503784179688, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.002149817533791065, + "fcm_dpo/delta": 0.02008049003779888, + "fcm_dpo/margin": 177.32235717773438, + "fcm_dpo/q_t": 0.4105430245399475, + "grad_norm": 39.78499984741211, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.3661746680736542, + "logits/rejected": -0.3563402593135834, + "logps/chosen": -380.18463134765625, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -574.6954345703125, + "loss": 1.0887, + "margin_dpo/margin_mean": 177.3223419189453, + "margin_dpo/margin_std": 204.42782592773438, + "step": 310 + }, + { + "KL/chosen_KL_mean": -321.9461669921875, + "KL/mean": -426.3747253417969, + "KL/rejected_KL_mean": -530.8032836914062, + "KL/std": 257.24591064453125, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.0020915032364428043, + "fcm_dpo/delta": -0.042413707822561264, + "fcm_dpo/margin": 208.85711669921875, + "fcm_dpo/q_t": 0.40021204948425293, + "grad_norm": 24.94213104248047, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.3498092293739319, + "logits/rejected": -0.3274255692958832, + "logps/chosen": -374.97412109375, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -608.241455078125, + "loss": 1.0718, + "margin_dpo/margin_mean": 208.85711669921875, + "margin_dpo/margin_std": 273.5059814453125, + "step": 311 + }, + { + "KL/chosen_KL_mean": -303.92413330078125, + "KL/mean": -401.8299560546875, + "KL/rejected_KL_mean": -499.73583984375, + "KL/std": 232.06793212890625, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0020924038253724575, + "fcm_dpo/delta": -0.01191241666674614, + "fcm_dpo/margin": 195.81173706054688, + "fcm_dpo/q_t": 0.40753504633903503, + "grad_norm": 27.596067428588867, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.29156944155693054, + "logits/rejected": -0.28357312083244324, + "logps/chosen": -353.31634521484375, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -575.5286865234375, + "loss": 1.0897, + "margin_dpo/margin_mean": 195.81173706054688, + "margin_dpo/margin_std": 268.7630920410156, + "step": 312 + }, + { + "KL/chosen_KL_mean": -275.786376953125, + "KL/mean": -375.9600830078125, + "KL/rejected_KL_mean": -476.1337890625, + "KL/std": 242.50588989257812, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.002089242683723569, + "fcm_dpo/delta": -0.024312660098075867, + "fcm_dpo/margin": 200.3473663330078, + "fcm_dpo/q_t": 0.4029426574707031, + "grad_norm": 40.433441162109375, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.3443872332572937, + "logits/rejected": -0.343222051858902, + "logps/chosen": -325.93914794921875, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -562.5399780273438, + "loss": 1.0878, + "margin_dpo/margin_mean": 200.3473663330078, + "margin_dpo/margin_std": 263.2047119140625, + "step": 313 + }, + { + "KL/chosen_KL_mean": -288.5335693359375, + "KL/mean": -379.27203369140625, + "KL/rejected_KL_mean": -470.010498046875, + "KL/std": 220.5050811767578, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0020419247448444366, + "fcm_dpo/delta": 0.03265833854675293, + "fcm_dpo/margin": 181.4769287109375, + "fcm_dpo/q_t": 0.41662710905075073, + "grad_norm": 19.380903244018555, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.3946872055530548, + "logits/rejected": -0.39694398641586304, + "logps/chosen": -345.7711486816406, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -567.6070556640625, + "loss": 1.1261, + "margin_dpo/margin_mean": 181.4769287109375, + "margin_dpo/margin_std": 277.6301574707031, + "step": 314 + }, + { + "KL/chosen_KL_mean": -257.5688171386719, + "KL/mean": -341.4803466796875, + "KL/rejected_KL_mean": -425.391845703125, + "KL/std": 201.53536987304688, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.002112956950441003, + "fcm_dpo/delta": 0.05089100822806358, + "fcm_dpo/margin": 167.8230743408203, + "fcm_dpo/q_t": 0.41888198256492615, + "grad_norm": 18.809349060058594, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.3166372776031494, + "logits/rejected": -0.32127225399017334, + "logps/chosen": -306.638427734375, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -511.07275390625, + "loss": 1.1161, + "margin_dpo/margin_mean": 167.8230743408203, + "margin_dpo/margin_std": 224.7148895263672, + "step": 315 + }, + { + "KL/chosen_KL_mean": -251.60336303710938, + "KL/mean": -372.1722412109375, + "KL/rejected_KL_mean": -492.7410888671875, + "KL/std": 235.99069213867188, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0020087971352040768, + "fcm_dpo/delta": -0.10278855264186859, + "fcm_dpo/margin": 241.1376953125, + "fcm_dpo/q_t": 0.3866315484046936, + "grad_norm": 26.42735481262207, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.38842660188674927, + "logits/rejected": -0.3925820589065552, + "logps/chosen": -305.8641357421875, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -594.0225830078125, + "loss": 1.0227, + "margin_dpo/margin_mean": 241.1376953125, + "margin_dpo/margin_std": 258.8439636230469, + "step": 316 + }, + { + "KL/chosen_KL_mean": -257.68450927734375, + "KL/mean": -389.06988525390625, + "KL/rejected_KL_mean": -520.4552001953125, + "KL/std": 216.13650512695312, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.0019280568230897188, + "fcm_dpo/delta": -0.12530095875263214, + "fcm_dpo/margin": 262.77069091796875, + "fcm_dpo/q_t": 0.37883231043815613, + "grad_norm": 29.969520568847656, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.3766433596611023, + "logits/rejected": -0.37070465087890625, + "logps/chosen": -313.77874755859375, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -621.1542358398438, + "loss": 0.998, + "margin_dpo/margin_mean": 262.7706604003906, + "margin_dpo/margin_std": 243.6465301513672, + "step": 317 + }, + { + "KL/chosen_KL_mean": -284.79205322265625, + "KL/mean": -382.79974365234375, + "KL/rejected_KL_mean": -480.807373046875, + "KL/std": 221.40475463867188, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0019070704001933336, + "fcm_dpo/delta": 0.028559785336256027, + "fcm_dpo/margin": 196.01536560058594, + "fcm_dpo/q_t": 0.4132635295391083, + "grad_norm": 22.410829544067383, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.44056424498558044, + "logits/rejected": -0.41288483142852783, + "logps/chosen": -349.437744140625, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -563.5716552734375, + "loss": 1.097, + "margin_dpo/margin_mean": 196.015380859375, + "margin_dpo/margin_std": 240.416748046875, + "step": 318 + }, + { + "KL/chosen_KL_mean": -256.9649353027344, + "KL/mean": -377.6507873535156, + "KL/rejected_KL_mean": -498.33660888671875, + "KL/std": 249.43978881835938, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.0018547551007941365, + "fcm_dpo/delta": -0.05502013862133026, + "fcm_dpo/margin": 241.37167358398438, + "fcm_dpo/q_t": 0.39447957277297974, + "grad_norm": 20.84139633178711, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.4100165367126465, + "logits/rejected": -0.4241865873336792, + "logps/chosen": -306.34869384765625, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -612.2431640625, + "loss": 1.0446, + "margin_dpo/margin_mean": 241.37167358398438, + "margin_dpo/margin_std": 266.0156555175781, + "step": 319 + }, + { + "KL/chosen_KL_mean": -279.9457702636719, + "KL/mean": -406.4906005859375, + "KL/rejected_KL_mean": -533.0354614257812, + "KL/std": 257.4986572265625, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.001772782881744206, + "fcm_dpo/delta": -0.0574687235057354, + "fcm_dpo/margin": 253.0897216796875, + "fcm_dpo/q_t": 0.39480236172676086, + "grad_norm": 26.903053283691406, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.4227886199951172, + "logits/rejected": -0.42991065979003906, + "logps/chosen": -339.4506530761719, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -630.70263671875, + "loss": 1.0431, + "margin_dpo/margin_mean": 253.0897216796875, + "margin_dpo/margin_std": 277.18646240234375, + "step": 320 + }, + { + "KL/chosen_KL_mean": -349.6231384277344, + "KL/mean": -459.3961181640625, + "KL/rejected_KL_mean": -569.1691284179688, + "KL/std": 268.8038024902344, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.0017589819617569447, + "fcm_dpo/delta": 0.014311235398054123, + "fcm_dpo/margin": 219.54598999023438, + "fcm_dpo/q_t": 0.4111507534980774, + "grad_norm": 21.74950408935547, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.35835346579551697, + "logits/rejected": -0.3437199890613556, + "logps/chosen": -411.17181396484375, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -660.8101806640625, + "loss": 1.0999, + "margin_dpo/margin_mean": 219.54598999023438, + "margin_dpo/margin_std": 288.1590881347656, + "step": 321 + }, + { + "KL/chosen_KL_mean": -306.5440673828125, + "KL/mean": -425.08251953125, + "KL/rejected_KL_mean": -543.6209716796875, + "KL/std": 237.18569946289062, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.001758311758749187, + "fcm_dpo/delta": -0.01994149014353752, + "fcm_dpo/margin": 237.07687377929688, + "fcm_dpo/q_t": 0.40169191360473633, + "grad_norm": 20.08477210998535, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.40985506772994995, + "logits/rejected": -0.4100232720375061, + "logps/chosen": -363.8341064453125, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -639.3709106445312, + "loss": 1.0608, + "margin_dpo/margin_mean": 237.07687377929688, + "margin_dpo/margin_std": 255.31802368164062, + "step": 322 + }, + { + "KL/chosen_KL_mean": -337.42376708984375, + "KL/mean": -436.5365905761719, + "KL/rejected_KL_mean": -535.6494140625, + "KL/std": 241.28932189941406, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.001798543380573392, + "fcm_dpo/delta": 0.04876996576786041, + "fcm_dpo/margin": 198.22564697265625, + "fcm_dpo/q_t": 0.4175468683242798, + "grad_norm": 30.990968704223633, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.43184393644332886, + "logits/rejected": -0.4301251173019409, + "logps/chosen": -388.65771484375, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -610.7113037109375, + "loss": 1.1164, + "margin_dpo/margin_mean": 198.22564697265625, + "margin_dpo/margin_std": 260.9371643066406, + "step": 323 + }, + { + "KL/chosen_KL_mean": -387.80419921875, + "KL/mean": -483.3709716796875, + "KL/rejected_KL_mean": -578.937744140625, + "KL/std": 262.6821594238281, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.0018935281550511718, + "fcm_dpo/delta": 0.03825587034225464, + "fcm_dpo/margin": 191.1335906982422, + "fcm_dpo/q_t": 0.41919389367103577, + "grad_norm": 43.27192687988281, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.43667468428611755, + "logits/rejected": -0.4218607544898987, + "logps/chosen": -452.9393310546875, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -665.415283203125, + "loss": 1.1418, + "margin_dpo/margin_mean": 191.1335906982422, + "margin_dpo/margin_std": 303.1317138671875, + "step": 324 + }, + { + "KL/chosen_KL_mean": -308.06524658203125, + "KL/mean": -390.8018798828125, + "KL/rejected_KL_mean": -473.53851318359375, + "KL/std": 231.8892059326172, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.001941892085596919, + "fcm_dpo/delta": 0.08667756617069244, + "fcm_dpo/margin": 165.4732208251953, + "fcm_dpo/q_t": 0.42669257521629333, + "grad_norm": 40.36817932128906, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.4763960838317871, + "logits/rejected": -0.4577608108520508, + "logps/chosen": -364.2808532714844, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -543.6243896484375, + "loss": 1.1456, + "margin_dpo/margin_mean": 165.47320556640625, + "margin_dpo/margin_std": 240.9735565185547, + "step": 325 + }, + { + "KL/chosen_KL_mean": -338.613037109375, + "KL/mean": -420.40509033203125, + "KL/rejected_KL_mean": -502.1971740722656, + "KL/std": 213.888671875, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0020220614969730377, + "fcm_dpo/delta": 0.07704143226146698, + "fcm_dpo/margin": 163.58416748046875, + "fcm_dpo/q_t": 0.4231676459312439, + "grad_norm": 57.26719665527344, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.4857342839241028, + "logits/rejected": -0.46466851234436035, + "logps/chosen": -411.3379821777344, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -582.0439453125, + "loss": 1.1277, + "margin_dpo/margin_mean": 163.58416748046875, + "margin_dpo/margin_std": 208.72296142578125, + "step": 326 + }, + { + "KL/chosen_KL_mean": -297.2885437011719, + "KL/mean": -418.62054443359375, + "KL/rejected_KL_mean": -539.9525146484375, + "KL/std": 228.9564971923828, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0019550384022295475, + "fcm_dpo/delta": -0.08856247365474701, + "fcm_dpo/margin": 242.6639404296875, + "fcm_dpo/q_t": 0.38669252395629883, + "grad_norm": 54.44054412841797, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.45049387216567993, + "logits/rejected": -0.43925124406814575, + "logps/chosen": -366.4229736328125, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -651.88623046875, + "loss": 1.0253, + "margin_dpo/margin_mean": 242.6639404296875, + "margin_dpo/margin_std": 253.40536499023438, + "step": 327 + }, + { + "KL/chosen_KL_mean": -311.56207275390625, + "KL/mean": -423.44635009765625, + "KL/rejected_KL_mean": -535.3306884765625, + "KL/std": 247.43212890625, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0019241906702518463, + "fcm_dpo/delta": -0.03506749868392944, + "fcm_dpo/margin": 223.76858520507812, + "fcm_dpo/q_t": 0.40035998821258545, + "grad_norm": 28.20893669128418, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.4424739480018616, + "logits/rejected": -0.4483252167701721, + "logps/chosen": -371.249267578125, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -626.1856689453125, + "loss": 1.0656, + "margin_dpo/margin_mean": 223.76858520507812, + "margin_dpo/margin_std": 274.401123046875, + "step": 328 + }, + { + "KL/chosen_KL_mean": -356.55242919921875, + "KL/mean": -474.0057373046875, + "KL/rejected_KL_mean": -591.458984375, + "KL/std": 279.4662170410156, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.0018408589530736208, + "fcm_dpo/delta": -0.04030177369713783, + "fcm_dpo/margin": 234.9065399169922, + "fcm_dpo/q_t": 0.3987047076225281, + "grad_norm": 25.61243438720703, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.46585187315940857, + "logits/rejected": -0.45958662033081055, + "logps/chosen": -421.79864501953125, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -692.15673828125, + "loss": 1.0744, + "margin_dpo/margin_mean": 234.90652465820312, + "margin_dpo/margin_std": 296.1156921386719, + "step": 329 + }, + { + "KL/chosen_KL_mean": -297.80157470703125, + "KL/mean": -420.50982666015625, + "KL/rejected_KL_mean": -543.2181396484375, + "KL/std": 255.82525634765625, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0018166296649724245, + "fcm_dpo/delta": -0.05299849808216095, + "fcm_dpo/margin": 245.41651916503906, + "fcm_dpo/q_t": 0.3957037329673767, + "grad_norm": 24.3403377532959, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.45932671427726746, + "logits/rejected": -0.45988693833351135, + "logps/chosen": -344.7999267578125, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -630.094970703125, + "loss": 1.0495, + "margin_dpo/margin_mean": 245.41653442382812, + "margin_dpo/margin_std": 280.8428955078125, + "step": 330 + }, + { + "KL/chosen_KL_mean": -324.4953308105469, + "KL/mean": -445.4620666503906, + "KL/rejected_KL_mean": -566.4288330078125, + "KL/std": 213.71218872070312, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.0017785282107070088, + "fcm_dpo/delta": -0.034743767231702805, + "fcm_dpo/margin": 241.93350219726562, + "fcm_dpo/q_t": 0.3970544934272766, + "grad_norm": 23.805938720703125, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.5145645141601562, + "logits/rejected": -0.5147561430931091, + "logps/chosen": -375.01959228515625, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -655.4442749023438, + "loss": 1.0353, + "margin_dpo/margin_mean": 241.93350219726562, + "margin_dpo/margin_std": 213.17160034179688, + "step": 331 + }, + { + "KL/chosen_KL_mean": -332.9132385253906, + "KL/mean": -416.395263671875, + "KL/rejected_KL_mean": -499.87725830078125, + "KL/std": 217.03173828125, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.0018567675724625587, + "fcm_dpo/delta": 0.09832817316055298, + "fcm_dpo/margin": 166.96405029296875, + "fcm_dpo/q_t": 0.42977607250213623, + "grad_norm": 21.644887924194336, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.46088725328445435, + "logits/rejected": -0.44053542613983154, + "logps/chosen": -382.093505859375, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -576.3624267578125, + "loss": 1.1494, + "margin_dpo/margin_mean": 166.96405029296875, + "margin_dpo/margin_std": 247.17556762695312, + "step": 332 + }, + { + "KL/chosen_KL_mean": -355.0767517089844, + "KL/mean": -469.5586242675781, + "KL/rejected_KL_mean": -584.0404663085938, + "KL/std": 267.2508544921875, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.001818150281906128, + "fcm_dpo/delta": -0.022109229117631912, + "fcm_dpo/margin": 228.9636993408203, + "fcm_dpo/q_t": 0.40483951568603516, + "grad_norm": 39.82794189453125, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.5101666450500488, + "logits/rejected": -0.5091091394424438, + "logps/chosen": -418.83251953125, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -679.0845947265625, + "loss": 1.0991, + "margin_dpo/margin_mean": 228.9636993408203, + "margin_dpo/margin_std": 325.2781677246094, + "step": 333 + }, + { + "KL/chosen_KL_mean": -332.78668212890625, + "KL/mean": -435.6429443359375, + "KL/rejected_KL_mean": -538.4992065429688, + "KL/std": 289.2935485839844, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0018572642002254725, + "fcm_dpo/delta": 0.020202260464429855, + "fcm_dpo/margin": 205.71255493164062, + "fcm_dpo/q_t": 0.4141433835029602, + "grad_norm": 22.896038055419922, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.46737080812454224, + "logits/rejected": -0.46773988008499146, + "logps/chosen": -399.76641845703125, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -633.816162109375, + "loss": 1.1258, + "margin_dpo/margin_mean": 205.71253967285156, + "margin_dpo/margin_std": 323.8262939453125, + "step": 334 + }, + { + "KL/chosen_KL_mean": -357.0669860839844, + "KL/mean": -439.2287902832031, + "KL/rejected_KL_mean": -521.3905639648438, + "KL/std": 267.3389892578125, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.0019713938236236572, + "fcm_dpo/delta": 0.08115716278553009, + "fcm_dpo/margin": 164.3235626220703, + "fcm_dpo/q_t": 0.427487313747406, + "grad_norm": 26.389087677001953, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.5104948282241821, + "logits/rejected": -0.49401405453681946, + "logps/chosen": -419.6094665527344, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -609.00830078125, + "loss": 1.1642, + "margin_dpo/margin_mean": 164.32354736328125, + "margin_dpo/margin_std": 283.85467529296875, + "step": 335 + }, + { + "KL/chosen_KL_mean": -356.31683349609375, + "KL/mean": -483.1954345703125, + "KL/rejected_KL_mean": -610.0740966796875, + "KL/std": 297.5561828613281, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.0019030815456062555, + "fcm_dpo/delta": -0.09671995788812637, + "fcm_dpo/margin": 253.75726318359375, + "fcm_dpo/q_t": 0.3906264305114746, + "grad_norm": 25.84535789489746, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.4441474676132202, + "logits/rejected": -0.44615668058395386, + "logps/chosen": -410.8479919433594, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -714.478271484375, + "loss": 1.0616, + "margin_dpo/margin_mean": 253.75726318359375, + "margin_dpo/margin_std": 347.39373779296875, + "step": 336 + }, + { + "KL/chosen_KL_mean": -329.84710693359375, + "KL/mean": -458.6802978515625, + "KL/rejected_KL_mean": -587.513427734375, + "KL/std": 246.65399169921875, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.0017828859854489565, + "fcm_dpo/delta": -0.0700145959854126, + "fcm_dpo/margin": 257.6663513183594, + "fcm_dpo/q_t": 0.3926036059856415, + "grad_norm": 19.696969985961914, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.4900759756565094, + "logits/rejected": -0.4794987738132477, + "logps/chosen": -394.975830078125, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -689.240478515625, + "loss": 1.0411, + "margin_dpo/margin_mean": 257.6663513183594, + "margin_dpo/margin_std": 292.0854797363281, + "step": 337 + }, + { + "KL/chosen_KL_mean": -291.3658752441406, + "KL/mean": -414.66558837890625, + "KL/rejected_KL_mean": -537.9652709960938, + "KL/std": 233.88966369628906, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.001743541331961751, + "fcm_dpo/delta": -0.03468111529946327, + "fcm_dpo/margin": 246.59938049316406, + "fcm_dpo/q_t": 0.39797914028167725, + "grad_norm": 21.53237533569336, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.4668804407119751, + "logits/rejected": -0.4627712368965149, + "logps/chosen": -349.78857421875, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -627.0338134765625, + "loss": 1.045, + "margin_dpo/margin_mean": 246.59939575195312, + "margin_dpo/margin_std": 245.27127075195312, + "step": 338 + }, + { + "KL/chosen_KL_mean": -312.69110107421875, + "KL/mean": -430.07891845703125, + "KL/rejected_KL_mean": -547.4667358398438, + "KL/std": 245.323974609375, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0017699088202789426, + "fcm_dpo/delta": -0.02178262174129486, + "fcm_dpo/margin": 234.77566528320312, + "fcm_dpo/q_t": 0.40182018280029297, + "grad_norm": 21.18806266784668, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.5156516432762146, + "logits/rejected": -0.5041229724884033, + "logps/chosen": -372.6864013671875, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -651.377685546875, + "loss": 1.0673, + "margin_dpo/margin_mean": 234.77566528320312, + "margin_dpo/margin_std": 247.67764282226562, + "step": 339 + }, + { + "KL/chosen_KL_mean": -322.544189453125, + "KL/mean": -427.60931396484375, + "KL/rejected_KL_mean": -532.6744384765625, + "KL/std": 239.99166870117188, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0017336560413241386, + "fcm_dpo/delta": 0.0398600697517395, + "fcm_dpo/margin": 210.1302490234375, + "fcm_dpo/q_t": 0.4163724184036255, + "grad_norm": 26.00166130065918, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.4009581208229065, + "logits/rejected": -0.38124585151672363, + "logps/chosen": -375.3743896484375, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -605.7816772460938, + "loss": 1.1145, + "margin_dpo/margin_mean": 210.1302490234375, + "margin_dpo/margin_std": 288.097412109375, + "step": 340 + }, + { + "KL/chosen_KL_mean": -314.5205078125, + "KL/mean": -439.66558837890625, + "KL/rejected_KL_mean": -564.8106689453125, + "KL/std": 260.55419921875, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0017149357590824366, + "fcm_dpo/delta": -0.034785643219947815, + "fcm_dpo/margin": 250.2902069091797, + "fcm_dpo/q_t": 0.3993799090385437, + "grad_norm": 25.704483032226562, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.3656594753265381, + "logits/rejected": -0.366064190864563, + "logps/chosen": -362.4202880859375, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -666.62060546875, + "loss": 1.0553, + "margin_dpo/margin_mean": 250.29019165039062, + "margin_dpo/margin_std": 271.3633117675781, + "step": 341 + }, + { + "KL/chosen_KL_mean": -311.4330749511719, + "KL/mean": -415.94781494140625, + "KL/rejected_KL_mean": -520.4625854492188, + "KL/std": 240.28048706054688, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0017700039315968752, + "fcm_dpo/delta": 0.03236134350299835, + "fcm_dpo/margin": 209.02951049804688, + "fcm_dpo/q_t": 0.4146597981452942, + "grad_norm": 28.696870803833008, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.4587517976760864, + "logits/rejected": -0.43348491191864014, + "logps/chosen": -383.4297180175781, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -613.0521850585938, + "loss": 1.1104, + "margin_dpo/margin_mean": 209.02951049804688, + "margin_dpo/margin_std": 284.8971862792969, + "step": 342 + }, + { + "KL/chosen_KL_mean": -299.84820556640625, + "KL/mean": -426.9278564453125, + "KL/rejected_KL_mean": -554.0075073242188, + "KL/std": 243.48492431640625, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.0017194761894643307, + "fcm_dpo/delta": -0.04282538592815399, + "fcm_dpo/margin": 254.15933227539062, + "fcm_dpo/q_t": 0.39487528800964355, + "grad_norm": 21.673538208007812, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.455980509519577, + "logits/rejected": -0.46136072278022766, + "logps/chosen": -354.2538146972656, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -665.0489501953125, + "loss": 1.0299, + "margin_dpo/margin_mean": 254.15933227539062, + "margin_dpo/margin_std": 215.90309143066406, + "step": 343 + }, + { + "KL/chosen_KL_mean": -296.745361328125, + "KL/mean": -412.1114501953125, + "KL/rejected_KL_mean": -527.4775390625, + "KL/std": 275.8745422363281, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.0017548566684126854, + "fcm_dpo/delta": -0.010991938412189484, + "fcm_dpo/margin": 230.73214721679688, + "fcm_dpo/q_t": 0.40747278928756714, + "grad_norm": 23.348596572875977, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.45125705003738403, + "logits/rejected": -0.45269471406936646, + "logps/chosen": -350.71002197265625, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -618.1008911132812, + "loss": 1.0852, + "margin_dpo/margin_mean": 230.73214721679688, + "margin_dpo/margin_std": 276.1148681640625, + "step": 344 + }, + { + "KL/chosen_KL_mean": -355.4430236816406, + "KL/mean": -481.6751403808594, + "KL/rejected_KL_mean": -607.9072265625, + "KL/std": 260.40435791015625, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0016906562959775329, + "fcm_dpo/delta": -0.03089074045419693, + "fcm_dpo/margin": 252.46421813964844, + "fcm_dpo/q_t": 0.3995891213417053, + "grad_norm": 20.64918327331543, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.4609513282775879, + "logits/rejected": -0.4600338041782379, + "logps/chosen": -417.12872314453125, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -707.397705078125, + "loss": 1.0578, + "margin_dpo/margin_mean": 252.46421813964844, + "margin_dpo/margin_std": 280.94976806640625, + "step": 345 + }, + { + "KL/chosen_KL_mean": -357.3460693359375, + "KL/mean": -475.60333251953125, + "KL/rejected_KL_mean": -593.860595703125, + "KL/std": 255.70980834960938, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.001671666745096445, + "fcm_dpo/delta": 0.005235597491264343, + "fcm_dpo/margin": 236.51458740234375, + "fcm_dpo/q_t": 0.40655505657196045, + "grad_norm": 20.04353141784668, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.46441325545310974, + "logits/rejected": -0.4553294777870178, + "logps/chosen": -412.602294921875, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -671.2759399414062, + "loss": 1.0787, + "margin_dpo/margin_mean": 236.51458740234375, + "margin_dpo/margin_std": 271.9715576171875, + "step": 346 + }, + { + "KL/chosen_KL_mean": -356.90362548828125, + "KL/mean": -470.62115478515625, + "KL/rejected_KL_mean": -584.3387451171875, + "KL/std": 268.69818115234375, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.0016904667718335986, + "fcm_dpo/delta": 0.017533594742417336, + "fcm_dpo/margin": 227.4351043701172, + "fcm_dpo/q_t": 0.41143548488616943, + "grad_norm": 22.72960090637207, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.46594953536987305, + "logits/rejected": -0.4611964225769043, + "logps/chosen": -414.4698486328125, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -676.69384765625, + "loss": 1.0996, + "margin_dpo/margin_mean": 227.43511962890625, + "margin_dpo/margin_std": 302.37640380859375, + "step": 347 + }, + { + "KL/chosen_KL_mean": -317.7919921875, + "KL/mean": -417.72503662109375, + "KL/rejected_KL_mean": -517.6580810546875, + "KL/std": 239.55807495117188, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.001714209676720202, + "fcm_dpo/delta": 0.06312470883131027, + "fcm_dpo/margin": 199.86610412597656, + "fcm_dpo/q_t": 0.42203137278556824, + "grad_norm": 22.838912963867188, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.5276233553886414, + "logits/rejected": -0.5287749767303467, + "logps/chosen": -374.10968017578125, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -606.7965087890625, + "loss": 1.1295, + "margin_dpo/margin_mean": 199.86610412597656, + "margin_dpo/margin_std": 276.6788024902344, + "step": 348 + }, + { + "KL/chosen_KL_mean": -304.62628173828125, + "KL/mean": -416.69232177734375, + "KL/rejected_KL_mean": -528.7584228515625, + "KL/std": 257.4140319824219, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.001781124621629715, + "fcm_dpo/delta": -0.00019471347332000732, + "fcm_dpo/margin": 224.13214111328125, + "fcm_dpo/q_t": 0.40955832600593567, + "grad_norm": 21.850996017456055, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.5128347873687744, + "logits/rejected": -0.5150008201599121, + "logps/chosen": -362.65179443359375, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -626.2635498046875, + "loss": 1.1022, + "margin_dpo/margin_mean": 224.13214111328125, + "margin_dpo/margin_std": 318.174072265625, + "step": 349 + }, + { + "KL/chosen_KL_mean": -328.1751708984375, + "KL/mean": -439.5986022949219, + "KL/rejected_KL_mean": -551.0220336914062, + "KL/std": 258.98297119140625, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.0017465191194787621, + "fcm_dpo/delta": 0.011304561048746109, + "fcm_dpo/margin": 222.84689331054688, + "fcm_dpo/q_t": 0.41017258167266846, + "grad_norm": 29.956239700317383, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.5287673473358154, + "logits/rejected": -0.5353476405143738, + "logps/chosen": -392.5056457519531, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -640.8936767578125, + "loss": 1.1105, + "margin_dpo/margin_mean": 222.84689331054688, + "margin_dpo/margin_std": 323.1487731933594, + "step": 350 + }, + { + "KL/chosen_KL_mean": -298.2241516113281, + "KL/mean": -432.45611572265625, + "KL/rejected_KL_mean": -566.6881103515625, + "KL/std": 291.6300048828125, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.001702027628198266, + "fcm_dpo/delta": -0.067390576004982, + "fcm_dpo/margin": 268.4639892578125, + "fcm_dpo/q_t": 0.39597606658935547, + "grad_norm": 34.303070068359375, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.5366276502609253, + "logits/rejected": -0.5398092865943909, + "logps/chosen": -358.8962707519531, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -668.2535400390625, + "loss": 1.0613, + "margin_dpo/margin_mean": 268.4639587402344, + "margin_dpo/margin_std": 349.1943359375, + "step": 351 + }, + { + "KL/chosen_KL_mean": -357.3224792480469, + "KL/mean": -439.783935546875, + "KL/rejected_KL_mean": -522.245361328125, + "KL/std": 273.31414794921875, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.0017764794174581766, + "fcm_dpo/delta": 0.1178225725889206, + "fcm_dpo/margin": 164.9228973388672, + "fcm_dpo/q_t": 0.43483996391296387, + "grad_norm": 40.680362701416016, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.5054371356964111, + "logits/rejected": -0.4717298448085785, + "logps/chosen": -428.26593017578125, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -598.8873291015625, + "loss": 1.1853, + "margin_dpo/margin_mean": 164.92288208007812, + "margin_dpo/margin_std": 303.029541015625, + "step": 352 + }, + { + "KL/chosen_KL_mean": -317.82281494140625, + "KL/mean": -427.15869140625, + "KL/rejected_KL_mean": -536.4945678710938, + "KL/std": 255.03466796875, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.0017993391957134008, + "fcm_dpo/delta": 0.005468603223562241, + "fcm_dpo/margin": 218.67178344726562, + "fcm_dpo/q_t": 0.4100494384765625, + "grad_norm": 25.425094604492188, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.5417718887329102, + "logits/rejected": -0.5283914804458618, + "logps/chosen": -376.2181396484375, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -616.830078125, + "loss": 1.1009, + "margin_dpo/margin_mean": 218.67178344726562, + "margin_dpo/margin_std": 295.93804931640625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -265.94793701171875, + "KL/mean": -383.2909240722656, + "KL/rejected_KL_mean": -500.6339416503906, + "KL/std": 243.97955322265625, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.001802937826141715, + "fcm_dpo/delta": -0.0266977958381176, + "fcm_dpo/margin": 234.68600463867188, + "fcm_dpo/q_t": 0.4004340171813965, + "grad_norm": 30.113487243652344, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.5027167201042175, + "logits/rejected": -0.491599977016449, + "logps/chosen": -325.75091552734375, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -589.3914794921875, + "loss": 1.0558, + "margin_dpo/margin_mean": 234.68601989746094, + "margin_dpo/margin_std": 251.94664001464844, + "step": 354 + }, + { + "KL/chosen_KL_mean": -265.74615478515625, + "KL/mean": -383.51519775390625, + "KL/rejected_KL_mean": -501.2842712402344, + "KL/std": 243.97427368164062, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0017716080183163285, + "fcm_dpo/delta": -0.02039785496890545, + "fcm_dpo/margin": 235.53810119628906, + "fcm_dpo/q_t": 0.40127843618392944, + "grad_norm": 41.493717193603516, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.5693776607513428, + "logits/rejected": -0.5621850490570068, + "logps/chosen": -319.8746337890625, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -583.6903076171875, + "loss": 1.0581, + "margin_dpo/margin_mean": 235.53810119628906, + "margin_dpo/margin_std": 247.4508056640625, + "step": 355 + }, + { + "KL/chosen_KL_mean": -335.10052490234375, + "KL/mean": -398.54974365234375, + "KL/rejected_KL_mean": -461.9989929199219, + "KL/std": 258.75146484375, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.0018006841419264674, + "fcm_dpo/delta": 0.03772455081343651, + "fcm_dpo/margin": 126.89848327636719, + "fcm_dpo/q_t": 0.44904446601867676, + "grad_norm": 32.35127258300781, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.5015488266944885, + "logits/rejected": -0.4796242117881775, + "logps/chosen": -399.7742919921875, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -537.8982543945312, + "loss": 1.2508, + "margin_dpo/margin_mean": 126.89848327636719, + "margin_dpo/margin_std": 321.12939453125, + "step": 356 + }, + { + "KL/chosen_KL_mean": -298.8155517578125, + "KL/mean": -402.3448181152344, + "KL/rejected_KL_mean": -505.8740234375, + "KL/std": 254.60498046875, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.0018197721801698208, + "fcm_dpo/delta": 0.025433188304305077, + "fcm_dpo/margin": 207.0584716796875, + "fcm_dpo/q_t": 0.4126337766647339, + "grad_norm": 31.602190017700195, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.5226550102233887, + "logits/rejected": -0.5116690993309021, + "logps/chosen": -351.5413818359375, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -592.7152099609375, + "loss": 1.1057, + "margin_dpo/margin_mean": 207.0584716796875, + "margin_dpo/margin_std": 273.7800598144531, + "step": 357 + }, + { + "KL/chosen_KL_mean": -269.4600524902344, + "KL/mean": -370.3817138671875, + "KL/rejected_KL_mean": -471.3033447265625, + "KL/std": 252.98434448242188, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0018717790953814983, + "fcm_dpo/delta": 0.024991383776068687, + "fcm_dpo/margin": 201.8433074951172, + "fcm_dpo/q_t": 0.41424447298049927, + "grad_norm": 29.80463409423828, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.4871600866317749, + "logits/rejected": -0.4676273763179779, + "logps/chosen": -332.6654968261719, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -559.6766357421875, + "loss": 1.1065, + "margin_dpo/margin_mean": 201.84329223632812, + "margin_dpo/margin_std": 277.7925720214844, + "step": 358 + }, + { + "KL/chosen_KL_mean": -311.064697265625, + "KL/mean": -430.10516357421875, + "KL/rejected_KL_mean": -549.1456298828125, + "KL/std": 253.67189025878906, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0018316541099920869, + "fcm_dpo/delta": -0.0420210063457489, + "fcm_dpo/margin": 238.08090209960938, + "fcm_dpo/q_t": 0.4000629484653473, + "grad_norm": 28.996294021606445, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.45488864183425903, + "logits/rejected": -0.440914124250412, + "logps/chosen": -367.4349060058594, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -631.3193359375, + "loss": 1.0686, + "margin_dpo/margin_mean": 238.08090209960938, + "margin_dpo/margin_std": 301.75921630859375, + "step": 359 + }, + { + "KL/chosen_KL_mean": -304.15875244140625, + "KL/mean": -397.9549865722656, + "KL/rejected_KL_mean": -491.7511901855469, + "KL/std": 211.96435546875, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.0018809980247169733, + "fcm_dpo/delta": 0.05237477272748947, + "fcm_dpo/margin": 187.5924530029297, + "fcm_dpo/q_t": 0.4174148738384247, + "grad_norm": 23.43485450744629, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.4171748757362366, + "logits/rejected": -0.40131115913391113, + "logps/chosen": -355.619140625, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -561.590087890625, + "loss": 1.102, + "margin_dpo/margin_mean": 187.5924530029297, + "margin_dpo/margin_std": 210.81942749023438, + "step": 360 + }, + { + "KL/chosen_KL_mean": -335.46392822265625, + "KL/mean": -425.03302001953125, + "KL/rejected_KL_mean": -514.6021118164062, + "KL/std": 248.8665313720703, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0019513891311362386, + "fcm_dpo/delta": 0.05541447550058365, + "fcm_dpo/margin": 179.13819885253906, + "fcm_dpo/q_t": 0.4199727475643158, + "grad_norm": 31.253250122070312, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.45148319005966187, + "logits/rejected": -0.4474943280220032, + "logps/chosen": -389.33343505859375, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -605.371337890625, + "loss": 1.1326, + "margin_dpo/margin_mean": 179.13819885253906, + "margin_dpo/margin_std": 269.49468994140625, + "step": 361 + }, + { + "KL/chosen_KL_mean": -285.95184326171875, + "KL/mean": -433.30718994140625, + "KL/rejected_KL_mean": -580.66259765625, + "KL/std": 261.23876953125, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.001797789242118597, + "fcm_dpo/delta": -0.15786853432655334, + "fcm_dpo/margin": 294.71075439453125, + "fcm_dpo/q_t": 0.37356308102607727, + "grad_norm": 24.21192741394043, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.4578719139099121, + "logits/rejected": -0.4666714668273926, + "logps/chosen": -344.5909118652344, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -686.2445068359375, + "loss": 0.9806, + "margin_dpo/margin_mean": 294.71075439453125, + "margin_dpo/margin_std": 275.6654357910156, + "step": 362 + }, + { + "KL/chosen_KL_mean": -280.5108642578125, + "KL/mean": -423.5561828613281, + "KL/rejected_KL_mean": -566.6015014648438, + "KL/std": 259.78265380859375, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0017072279006242752, + "fcm_dpo/delta": -0.10325239598751068, + "fcm_dpo/margin": 286.090576171875, + "fcm_dpo/q_t": 0.3843806982040405, + "grad_norm": 26.743206024169922, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.4378637671470642, + "logits/rejected": -0.4329283833503723, + "logps/chosen": -325.06927490234375, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -641.2964477539062, + "loss": 1.012, + "margin_dpo/margin_mean": 286.090576171875, + "margin_dpo/margin_std": 286.5257873535156, + "step": 363 + }, + { + "KL/chosen_KL_mean": -305.91583251953125, + "KL/mean": -416.8153076171875, + "KL/rejected_KL_mean": -527.7147827148438, + "KL/std": 254.28158569335938, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0017074677161872387, + "fcm_dpo/delta": 0.022615114226937294, + "fcm_dpo/margin": 221.79896545410156, + "fcm_dpo/q_t": 0.41215628385543823, + "grad_norm": 22.465736389160156, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.46301624178886414, + "logits/rejected": -0.4743831753730774, + "logps/chosen": -354.8104248046875, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -619.1105346679688, + "loss": 1.1035, + "margin_dpo/margin_mean": 221.79896545410156, + "margin_dpo/margin_std": 292.4964599609375, + "step": 364 + }, + { + "KL/chosen_KL_mean": -305.3367004394531, + "KL/mean": -421.4215087890625, + "KL/rejected_KL_mean": -537.50634765625, + "KL/std": 266.6111145019531, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.001697975443676114, + "fcm_dpo/delta": 0.006551774684339762, + "fcm_dpo/margin": 232.169677734375, + "fcm_dpo/q_t": 0.40931108593940735, + "grad_norm": 25.95713996887207, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.4985828697681427, + "logits/rejected": -0.5110834240913391, + "logps/chosen": -356.8294372558594, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -630.2080078125, + "loss": 1.0821, + "margin_dpo/margin_mean": 232.16966247558594, + "margin_dpo/margin_std": 277.564453125, + "step": 365 + }, + { + "KL/chosen_KL_mean": -297.9063720703125, + "KL/mean": -417.0794982910156, + "KL/rejected_KL_mean": -536.2525634765625, + "KL/std": 260.83770751953125, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0016744910972192883, + "fcm_dpo/delta": -0.0004605799913406372, + "fcm_dpo/margin": 238.34617614746094, + "fcm_dpo/q_t": 0.40801802277565, + "grad_norm": 18.043302536010742, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.4490824341773987, + "logits/rejected": -0.4660515785217285, + "logps/chosen": -342.626953125, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -619.56298828125, + "loss": 1.0924, + "margin_dpo/margin_mean": 238.34616088867188, + "margin_dpo/margin_std": 311.0103759765625, + "step": 366 + }, + { + "KL/chosen_KL_mean": -314.82867431640625, + "KL/mean": -414.3272399902344, + "KL/rejected_KL_mean": -513.8258056640625, + "KL/std": 233.58694458007812, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0017527085728943348, + "fcm_dpo/delta": 0.0565352737903595, + "fcm_dpo/margin": 198.99717712402344, + "fcm_dpo/q_t": 0.41837412118911743, + "grad_norm": 27.162324905395508, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.49326080083847046, + "logits/rejected": -0.4785804748535156, + "logps/chosen": -373.23406982421875, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -590.5771484375, + "loss": 1.1136, + "margin_dpo/margin_mean": 198.99717712402344, + "margin_dpo/margin_std": 249.590087890625, + "step": 367 + }, + { + "KL/chosen_KL_mean": -303.561279296875, + "KL/mean": -461.7544250488281, + "KL/rejected_KL_mean": -619.947509765625, + "KL/std": 273.23431396484375, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0016491047572344542, + "fcm_dpo/delta": -0.14441686868667603, + "fcm_dpo/margin": 316.38623046875, + "fcm_dpo/q_t": 0.37210506200790405, + "grad_norm": 30.41951560974121, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.49794042110443115, + "logits/rejected": -0.5089166164398193, + "logps/chosen": -348.0137939453125, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -718.5028076171875, + "loss": 0.9672, + "margin_dpo/margin_mean": 316.38623046875, + "margin_dpo/margin_std": 236.6722869873047, + "step": 368 + }, + { + "KL/chosen_KL_mean": -398.1829833984375, + "KL/mean": -482.0100402832031, + "KL/rejected_KL_mean": -565.8370971679688, + "KL/std": 271.72454833984375, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.001717425067909062, + "fcm_dpo/delta": 0.12066368013620377, + "fcm_dpo/margin": 167.6540985107422, + "fcm_dpo/q_t": 0.4341936409473419, + "grad_norm": 27.30344581604004, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.5581328868865967, + "logits/rejected": -0.5420875549316406, + "logps/chosen": -469.56451416015625, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -657.1329345703125, + "loss": 1.184, + "margin_dpo/margin_mean": 167.6540985107422, + "margin_dpo/margin_std": 304.3944396972656, + "step": 369 + }, + { + "KL/chosen_KL_mean": -426.78857421875, + "KL/mean": -500.9025573730469, + "KL/rejected_KL_mean": -575.0164794921875, + "KL/std": 289.44635009765625, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0017565288580954075, + "fcm_dpo/delta": 0.046290576457977295, + "fcm_dpo/margin": 148.22787475585938, + "fcm_dpo/q_t": 0.44084054231643677, + "grad_norm": 33.96835708618164, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.5518050789833069, + "logits/rejected": -0.5443004369735718, + "logps/chosen": -498.3960876464844, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -672.2762451171875, + "loss": 1.2213, + "margin_dpo/margin_mean": 148.22787475585938, + "margin_dpo/margin_std": 326.36639404296875, + "step": 370 + }, + { + "KL/chosen_KL_mean": -410.635498046875, + "KL/mean": -523.5191650390625, + "KL/rejected_KL_mean": -636.4027709960938, + "KL/std": 294.27691650390625, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0017921735998243093, + "fcm_dpo/delta": -0.005750877782702446, + "fcm_dpo/margin": 225.7672576904297, + "fcm_dpo/q_t": 0.4080857038497925, + "grad_norm": 35.65329360961914, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.56690913438797, + "logits/rejected": -0.5529294610023499, + "logps/chosen": -480.04998779296875, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -735.574951171875, + "loss": 1.1032, + "margin_dpo/margin_mean": 225.76724243164062, + "margin_dpo/margin_std": 328.3232421875, + "step": 371 + }, + { + "KL/chosen_KL_mean": -399.5024719238281, + "KL/mean": -532.6174926757812, + "KL/rejected_KL_mean": -665.7325439453125, + "KL/std": 331.44317626953125, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.0017226326745003462, + "fcm_dpo/delta": -0.0679711401462555, + "fcm_dpo/margin": 266.2300720214844, + "fcm_dpo/q_t": 0.3977503776550293, + "grad_norm": 31.05590057373047, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.5817974805831909, + "logits/rejected": -0.5693035125732422, + "logps/chosen": -461.3204650878906, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -744.2720336914062, + "loss": 1.0704, + "margin_dpo/margin_mean": 266.2300720214844, + "margin_dpo/margin_std": 377.0699462890625, + "step": 372 + }, + { + "KL/chosen_KL_mean": -426.08880615234375, + "KL/mean": -551.7238159179688, + "KL/rejected_KL_mean": -677.3587646484375, + "KL/std": 308.47491455078125, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.0017068360466510057, + "fcm_dpo/delta": -0.034556809812784195, + "fcm_dpo/margin": 251.27001953125, + "fcm_dpo/q_t": 0.3999958634376526, + "grad_norm": 28.885194778442383, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.5467664003372192, + "logits/rejected": -0.5298678874969482, + "logps/chosen": -490.305908203125, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -763.3184204101562, + "loss": 1.075, + "margin_dpo/margin_mean": 251.27001953125, + "margin_dpo/margin_std": 308.8395690917969, + "step": 373 + }, + { + "KL/chosen_KL_mean": -378.645751953125, + "KL/mean": -492.173095703125, + "KL/rejected_KL_mean": -605.7003173828125, + "KL/std": 333.42578125, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.0016566256526857615, + "fcm_dpo/delta": 0.024895425885915756, + "fcm_dpo/margin": 227.05455017089844, + "fcm_dpo/q_t": 0.41718772053718567, + "grad_norm": 26.81859016418457, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.6103962659835815, + "logits/rejected": -0.594162106513977, + "logps/chosen": -449.29595947265625, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -699.3405151367188, + "loss": 1.1391, + "margin_dpo/margin_mean": 227.05455017089844, + "margin_dpo/margin_std": 382.1032409667969, + "step": 374 + }, + { + "KL/chosen_KL_mean": -380.33349609375, + "KL/mean": -491.43072509765625, + "KL/rejected_KL_mean": -602.528076171875, + "KL/std": 268.87115478515625, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0016815003473311663, + "fcm_dpo/delta": 0.027468301355838776, + "fcm_dpo/margin": 222.1945037841797, + "fcm_dpo/q_t": 0.41399097442626953, + "grad_norm": 27.043115615844727, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.5385172367095947, + "logits/rejected": -0.5370956659317017, + "logps/chosen": -440.4136962890625, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -691.46630859375, + "loss": 1.1193, + "margin_dpo/margin_mean": 222.19451904296875, + "margin_dpo/margin_std": 316.20404052734375, + "step": 375 + }, + { + "KL/chosen_KL_mean": -355.98333740234375, + "KL/mean": -495.6485900878906, + "KL/rejected_KL_mean": -635.3138427734375, + "KL/std": 287.6280517578125, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0016569116851314902, + "fcm_dpo/delta": -0.07370490580797195, + "fcm_dpo/margin": 279.33050537109375, + "fcm_dpo/q_t": 0.39189913868904114, + "grad_norm": 25.121034622192383, + "learning_rate": 2.5e-07, + "logits/chosen": -0.55328369140625, + "logits/rejected": -0.5407309532165527, + "logps/chosen": -418.6436462402344, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -740.8404541015625, + "loss": 1.0469, + "margin_dpo/margin_mean": 279.33050537109375, + "margin_dpo/margin_std": 332.783935546875, + "step": 376 + }, + { + "KL/chosen_KL_mean": -349.88140869140625, + "KL/mean": -483.66864013671875, + "KL/rejected_KL_mean": -617.4559326171875, + "KL/std": 302.268310546875, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.001627064310014248, + "fcm_dpo/delta": -0.040655992925167084, + "fcm_dpo/margin": 267.57452392578125, + "fcm_dpo/q_t": 0.39833271503448486, + "grad_norm": 24.557861328125, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.4762452244758606, + "logits/rejected": -0.47602522373199463, + "logps/chosen": -404.360107421875, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -716.1592407226562, + "loss": 1.058, + "margin_dpo/margin_mean": 267.57452392578125, + "margin_dpo/margin_std": 311.4625244140625, + "step": 377 + }, + { + "KL/chosen_KL_mean": -327.9580078125, + "KL/mean": -464.2567138671875, + "KL/rejected_KL_mean": -600.555419921875, + "KL/std": 264.3885498046875, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.0015813407953828573, + "fcm_dpo/delta": -0.03572680056095123, + "fcm_dpo/margin": 272.5974426269531, + "fcm_dpo/q_t": 0.3976179361343384, + "grad_norm": 28.108327865600586, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.4832850396633148, + "logits/rejected": -0.5009036064147949, + "logps/chosen": -372.9785461425781, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -688.6024169921875, + "loss": 1.0484, + "margin_dpo/margin_mean": 272.5974426269531, + "margin_dpo/margin_std": 283.22003173828125, + "step": 378 + }, + { + "KL/chosen_KL_mean": -355.2315673828125, + "KL/mean": -489.8199157714844, + "KL/rejected_KL_mean": -624.408203125, + "KL/std": 265.6845703125, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0015417186077684164, + "fcm_dpo/delta": -0.018481876701116562, + "fcm_dpo/margin": 269.1766662597656, + "fcm_dpo/q_t": 0.40319210290908813, + "grad_norm": 24.342618942260742, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.43572235107421875, + "logits/rejected": -0.45832598209381104, + "logps/chosen": -408.4136962890625, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -738.7083740234375, + "loss": 1.0715, + "margin_dpo/margin_mean": 269.1766662597656, + "margin_dpo/margin_std": 317.8804016113281, + "step": 379 + }, + { + "KL/chosen_KL_mean": -336.372802734375, + "KL/mean": -483.15008544921875, + "KL/rejected_KL_mean": -629.9274291992188, + "KL/std": 301.1836242675781, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0015245577087625861, + "fcm_dpo/delta": -0.05481116846203804, + "fcm_dpo/margin": 293.55462646484375, + "fcm_dpo/q_t": 0.3936833441257477, + "grad_norm": 23.968381881713867, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.4733444154262543, + "logits/rejected": -0.5023707151412964, + "logps/chosen": -387.725830078125, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -734.119140625, + "loss": 1.0461, + "margin_dpo/margin_mean": 293.55462646484375, + "margin_dpo/margin_std": 325.58978271484375, + "step": 380 + }, + { + "KL/chosen_KL_mean": -344.7323303222656, + "KL/mean": -448.1922607421875, + "KL/rejected_KL_mean": -551.6522216796875, + "KL/std": 250.53060913085938, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0015589288668707013, + "fcm_dpo/delta": 0.0859440490603447, + "fcm_dpo/margin": 206.91986083984375, + "fcm_dpo/q_t": 0.4266587495803833, + "grad_norm": 29.088565826416016, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.46495842933654785, + "logits/rejected": -0.4453268051147461, + "logps/chosen": -402.535400390625, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -630.87158203125, + "loss": 1.156, + "margin_dpo/margin_mean": 206.91986083984375, + "margin_dpo/margin_std": 337.11724853515625, + "step": 381 + }, + { + "KL/chosen_KL_mean": -333.6182556152344, + "KL/mean": -461.98455810546875, + "KL/rejected_KL_mean": -590.350830078125, + "KL/std": 241.47354125976562, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.001582136144861579, + "fcm_dpo/delta": -0.007043153513222933, + "fcm_dpo/margin": 256.732666015625, + "fcm_dpo/q_t": 0.40415507555007935, + "grad_norm": 25.747312545776367, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.5053166747093201, + "logits/rejected": -0.5117884874343872, + "logps/chosen": -399.6385498046875, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -701.06103515625, + "loss": 1.063, + "margin_dpo/margin_mean": 256.732666015625, + "margin_dpo/margin_std": 266.22235107421875, + "step": 382 + }, + { + "KL/chosen_KL_mean": -350.60723876953125, + "KL/mean": -475.28656005859375, + "KL/rejected_KL_mean": -599.9659423828125, + "KL/std": 273.14569091796875, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0015809867763891816, + "fcm_dpo/delta": 0.006530873477458954, + "fcm_dpo/margin": 249.35870361328125, + "fcm_dpo/q_t": 0.4085789918899536, + "grad_norm": 25.35655403137207, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.49040186405181885, + "logits/rejected": -0.4994901120662689, + "logps/chosen": -400.99871826171875, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -693.6818237304688, + "loss": 1.0994, + "margin_dpo/margin_mean": 249.35867309570312, + "margin_dpo/margin_std": 342.5252685546875, + "step": 383 + }, + { + "KL/chosen_KL_mean": -345.0385437011719, + "KL/mean": -447.2079162597656, + "KL/rejected_KL_mean": -549.3773193359375, + "KL/std": 243.29251098632812, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0016496088355779648, + "fcm_dpo/delta": 0.06874240189790726, + "fcm_dpo/margin": 204.3387908935547, + "fcm_dpo/q_t": 0.4205501079559326, + "grad_norm": 24.32337760925293, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.5029053688049316, + "logits/rejected": -0.49740487337112427, + "logps/chosen": -397.08465576171875, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -635.13818359375, + "loss": 1.1106, + "margin_dpo/margin_mean": 204.33877563476562, + "margin_dpo/margin_std": 223.48184204101562, + "step": 384 + }, + { + "KL/chosen_KL_mean": -325.19647216796875, + "KL/mean": -445.53045654296875, + "KL/rejected_KL_mean": -565.8643798828125, + "KL/std": 216.89447021484375, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.001653736107982695, + "fcm_dpo/delta": 0.002269826829433441, + "fcm_dpo/margin": 240.6679229736328, + "fcm_dpo/q_t": 0.4055292010307312, + "grad_norm": 29.387590408325195, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.5634757280349731, + "logits/rejected": -0.5416460037231445, + "logps/chosen": -390.7486572265625, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -643.6923217773438, + "loss": 1.0656, + "margin_dpo/margin_mean": 240.6679229736328, + "margin_dpo/margin_std": 240.91244506835938, + "step": 385 + }, + { + "KL/chosen_KL_mean": -341.10723876953125, + "KL/mean": -474.9710998535156, + "KL/rejected_KL_mean": -608.8349609375, + "KL/std": 274.450927734375, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0016267395112663507, + "fcm_dpo/delta": -0.04082644730806351, + "fcm_dpo/margin": 267.7276611328125, + "fcm_dpo/q_t": 0.398295134305954, + "grad_norm": 20.20743751525879, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.4894062876701355, + "logits/rejected": -0.4843745231628418, + "logps/chosen": -399.3291015625, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -701.162353515625, + "loss": 1.058, + "margin_dpo/margin_mean": 267.7276916503906, + "margin_dpo/margin_std": 314.4848327636719, + "step": 386 + }, + { + "KL/chosen_KL_mean": -378.54974365234375, + "KL/mean": -490.86309814453125, + "KL/rejected_KL_mean": -603.1763916015625, + "KL/std": 258.2947692871094, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0016245550941675901, + "fcm_dpo/delta": 0.03896220028400421, + "fcm_dpo/margin": 224.62669372558594, + "fcm_dpo/q_t": 0.4146435260772705, + "grad_norm": 27.515451431274414, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.5612543225288391, + "logits/rejected": -0.5425852537155151, + "logps/chosen": -444.96917724609375, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -695.3455810546875, + "loss": 1.0995, + "margin_dpo/margin_mean": 224.62667846679688, + "margin_dpo/margin_std": 262.4505615234375, + "step": 387 + }, + { + "KL/chosen_KL_mean": -361.38873291015625, + "KL/mean": -511.38482666015625, + "KL/rejected_KL_mean": -661.3810424804688, + "KL/std": 301.95050048828125, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0015752706676721573, + "fcm_dpo/delta": -0.08615969866514206, + "fcm_dpo/margin": 299.9922790527344, + "fcm_dpo/q_t": 0.39034122228622437, + "grad_norm": 31.809385299682617, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.48613548278808594, + "logits/rejected": -0.49910032749176025, + "logps/chosen": -411.5181884765625, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -765.8140869140625, + "loss": 1.0356, + "margin_dpo/margin_mean": 299.9922790527344, + "margin_dpo/margin_std": 344.7364501953125, + "step": 388 + }, + { + "KL/chosen_KL_mean": -401.0498046875, + "KL/mean": -529.2842407226562, + "KL/rejected_KL_mean": -657.5186767578125, + "KL/std": 298.679443359375, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.001562059624120593, + "fcm_dpo/delta": -0.0008359383791685104, + "fcm_dpo/margin": 256.4689025878906, + "fcm_dpo/q_t": 0.4074057638645172, + "grad_norm": 26.30559539794922, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.5535522699356079, + "logits/rejected": -0.5469101667404175, + "logps/chosen": -458.9563903808594, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -735.4332275390625, + "loss": 1.0887, + "margin_dpo/margin_mean": 256.4689025878906, + "margin_dpo/margin_std": 332.9835205078125, + "step": 389 + }, + { + "KL/chosen_KL_mean": -401.0078430175781, + "KL/mean": -522.4735107421875, + "KL/rejected_KL_mean": -643.9390869140625, + "KL/std": 297.29010009765625, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.0015696054324507713, + "fcm_dpo/delta": 0.02077120542526245, + "fcm_dpo/margin": 242.9313201904297, + "fcm_dpo/q_t": 0.41563481092453003, + "grad_norm": 23.253326416015625, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.48045140504837036, + "logits/rejected": -0.47344475984573364, + "logps/chosen": -450.2337646484375, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -729.46728515625, + "loss": 1.1242, + "margin_dpo/margin_mean": 242.93130493164062, + "margin_dpo/margin_std": 385.81768798828125, + "step": 390 + }, + { + "KL/chosen_KL_mean": -406.48260498046875, + "KL/mean": -481.1493835449219, + "KL/rejected_KL_mean": -555.816162109375, + "KL/std": 283.8955383300781, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0017056525684893131, + "fcm_dpo/delta": 0.15707972645759583, + "fcm_dpo/margin": 149.33358764648438, + "fcm_dpo/q_t": 0.4433702826499939, + "grad_norm": 40.65006637573242, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.49984246492385864, + "logits/rejected": -0.49244457483291626, + "logps/chosen": -470.812255859375, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -642.5543823242188, + "loss": 1.2099, + "margin_dpo/margin_mean": 149.33358764648438, + "margin_dpo/margin_std": 291.1400451660156, + "step": 391 + }, + { + "KL/chosen_KL_mean": -338.31121826171875, + "KL/mean": -489.0209655761719, + "KL/rejected_KL_mean": -639.730712890625, + "KL/std": 301.0827331542969, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0016687922179698944, + "fcm_dpo/delta": -0.12080799043178558, + "fcm_dpo/margin": 301.41949462890625, + "fcm_dpo/q_t": 0.3806690573692322, + "grad_norm": 23.98059844970703, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.468877375125885, + "logits/rejected": -0.46511754393577576, + "logps/chosen": -391.815185546875, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -742.0765380859375, + "loss": 0.9995, + "margin_dpo/margin_mean": 301.41949462890625, + "margin_dpo/margin_std": 287.8547668457031, + "step": 392 + }, + { + "KL/chosen_KL_mean": -341.0359802246094, + "KL/mean": -453.18865966796875, + "KL/rejected_KL_mean": -565.3413696289062, + "KL/std": 297.093017578125, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0016412805998697877, + "fcm_dpo/delta": 0.03581807762384415, + "fcm_dpo/margin": 224.30535888671875, + "fcm_dpo/q_t": 0.4194226861000061, + "grad_norm": 24.413087844848633, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.49346253275871277, + "logits/rejected": -0.4946703016757965, + "logps/chosen": -387.5098876953125, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -637.3101806640625, + "loss": 1.1346, + "margin_dpo/margin_mean": 224.30535888671875, + "margin_dpo/margin_std": 373.1012268066406, + "step": 393 + }, + { + "KL/chosen_KL_mean": -379.68658447265625, + "KL/mean": -500.4499206542969, + "KL/rejected_KL_mean": -621.2133178710938, + "KL/std": 310.21966552734375, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.0016519733471795917, + "fcm_dpo/delta": 0.0010132305324077606, + "fcm_dpo/margin": 241.52671813964844, + "fcm_dpo/q_t": 0.40879496932029724, + "grad_norm": 25.774150848388672, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.4709600508213043, + "logits/rejected": -0.4727855920791626, + "logps/chosen": -432.5981140136719, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -712.0359497070312, + "loss": 1.0914, + "margin_dpo/margin_mean": 241.52671813964844, + "margin_dpo/margin_std": 319.96893310546875, + "step": 394 + }, + { + "KL/chosen_KL_mean": -381.13262939453125, + "KL/mean": -512.487060546875, + "KL/rejected_KL_mean": -643.841552734375, + "KL/std": 304.2357177734375, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0016198453959077597, + "fcm_dpo/delta": -0.03043615072965622, + "fcm_dpo/margin": 262.7088623046875, + "fcm_dpo/q_t": 0.40265268087387085, + "grad_norm": 27.07144546508789, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.5321487188339233, + "logits/rejected": -0.5288753509521484, + "logps/chosen": -443.6787414550781, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -727.6241455078125, + "loss": 1.0808, + "margin_dpo/margin_mean": 262.7088623046875, + "margin_dpo/margin_std": 351.6431579589844, + "step": 395 + }, + { + "KL/chosen_KL_mean": -387.3375244140625, + "KL/mean": -505.85552978515625, + "KL/rejected_KL_mean": -624.3735961914062, + "KL/std": 303.82720947265625, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.0016323941526934505, + "fcm_dpo/delta": 0.014695589430630207, + "fcm_dpo/margin": 237.03604125976562, + "fcm_dpo/q_t": 0.40971770882606506, + "grad_norm": 31.21180534362793, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.533403754234314, + "logits/rejected": -0.5189059376716614, + "logps/chosen": -456.3334655761719, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -713.020263671875, + "loss": 1.086, + "margin_dpo/margin_mean": 237.03602600097656, + "margin_dpo/margin_std": 282.26318359375, + "step": 396 + }, + { + "KL/chosen_KL_mean": -360.510009765625, + "KL/mean": -513.9752197265625, + "KL/rejected_KL_mean": -667.4403686523438, + "KL/std": 300.31671142578125, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0015570519026368856, + "fcm_dpo/delta": -0.0934409499168396, + "fcm_dpo/margin": 306.930419921875, + "fcm_dpo/q_t": 0.38653814792633057, + "grad_norm": 30.107542037963867, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.48488834500312805, + "logits/rejected": -0.4924238324165344, + "logps/chosen": -421.78717041015625, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -770.5565185546875, + "loss": 1.0199, + "margin_dpo/margin_mean": 306.93035888671875, + "margin_dpo/margin_std": 312.76171875, + "step": 397 + }, + { + "KL/chosen_KL_mean": -380.28778076171875, + "KL/mean": -520.15771484375, + "KL/rejected_KL_mean": -660.0276489257812, + "KL/std": 307.3370666503906, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0015305501874536276, + "fcm_dpo/delta": -0.032348498702049255, + "fcm_dpo/margin": 279.7398681640625, + "fcm_dpo/q_t": 0.40199190378189087, + "grad_norm": 20.995561599731445, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.5499963760375977, + "logits/rejected": -0.5474686622619629, + "logps/chosen": -448.4393310546875, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -768.55126953125, + "loss": 1.0737, + "margin_dpo/margin_mean": 279.7398681640625, + "margin_dpo/margin_std": 363.536865234375, + "step": 398 + }, + { + "KL/chosen_KL_mean": -334.5681457519531, + "KL/mean": -452.637451171875, + "KL/rejected_KL_mean": -570.7067260742188, + "KL/std": 269.42364501953125, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0015373732894659042, + "fcm_dpo/delta": 0.041213102638721466, + "fcm_dpo/margin": 236.13858032226562, + "fcm_dpo/q_t": 0.4166143238544464, + "grad_norm": 25.833148956298828, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.4991453289985657, + "logits/rejected": -0.4608256220817566, + "logps/chosen": -395.45794677734375, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -648.6723022460938, + "loss": 1.1245, + "margin_dpo/margin_mean": 236.13856506347656, + "margin_dpo/margin_std": 346.589111328125, + "step": 399 + }, + { + "KL/chosen_KL_mean": -322.90533447265625, + "KL/mean": -496.75537109375, + "KL/rejected_KL_mean": -670.6054077148438, + "KL/std": 289.8807678222656, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0014637766871601343, + "fcm_dpo/delta": -0.13017460703849792, + "fcm_dpo/margin": 347.70013427734375, + "fcm_dpo/q_t": 0.3769122362136841, + "grad_norm": 23.40703582763672, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.5232506990432739, + "logits/rejected": -0.5229721069335938, + "logps/chosen": -386.54888916015625, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -775.8580932617188, + "loss": 0.9912, + "margin_dpo/margin_mean": 347.70013427734375, + "margin_dpo/margin_std": 318.1121826171875, + "step": 400 + }, + { + "KL/chosen_KL_mean": -376.1964111328125, + "KL/mean": -474.5458068847656, + "KL/rejected_KL_mean": -572.8951416015625, + "KL/std": 303.4837646484375, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0015107241924852133, + "fcm_dpo/delta": 0.11333553493022919, + "fcm_dpo/margin": 196.69879150390625, + "fcm_dpo/q_t": 0.4350917637348175, + "grad_norm": 26.16416358947754, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.48903053998947144, + "logits/rejected": -0.48064637184143066, + "logps/chosen": -433.3594665527344, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -656.6876220703125, + "loss": 1.2001, + "margin_dpo/margin_mean": 196.69876098632812, + "margin_dpo/margin_std": 405.4509582519531, + "step": 401 + }, + { + "KL/chosen_KL_mean": -276.8006591796875, + "KL/mean": -459.3830871582031, + "KL/rejected_KL_mean": -641.9654541015625, + "KL/std": 324.927001953125, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0014450090238824487, + "fcm_dpo/delta": -0.151978000998497, + "fcm_dpo/margin": 365.1648254394531, + "fcm_dpo/q_t": 0.3735431432723999, + "grad_norm": 22.905637741088867, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.5385646820068359, + "logits/rejected": -0.5310858488082886, + "logps/chosen": -327.541015625, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -723.0115356445312, + "loss": 0.9797, + "margin_dpo/margin_mean": 365.1648254394531, + "margin_dpo/margin_std": 335.25128173828125, + "step": 402 + }, + { + "KL/chosen_KL_mean": -303.8877868652344, + "KL/mean": -443.2480163574219, + "KL/rejected_KL_mean": -582.608154296875, + "KL/std": 303.9786682128906, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0014143368462100625, + "fcm_dpo/delta": 0.006570478901267052, + "fcm_dpo/margin": 278.720458984375, + "fcm_dpo/q_t": 0.40799298882484436, + "grad_norm": 23.390703201293945, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.5329214334487915, + "logits/rejected": -0.5339211225509644, + "logps/chosen": -351.03509521484375, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -659.8748168945312, + "loss": 1.0798, + "margin_dpo/margin_mean": 278.7204284667969, + "margin_dpo/margin_std": 319.8437805175781, + "step": 403 + }, + { + "KL/chosen_KL_mean": -338.0448303222656, + "KL/mean": -474.4947509765625, + "KL/rejected_KL_mean": -610.9447631835938, + "KL/std": 288.77886962890625, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0014204630861058831, + "fcm_dpo/delta": 0.013921055011451244, + "fcm_dpo/margin": 272.89990234375, + "fcm_dpo/q_t": 0.41045406460762024, + "grad_norm": 20.89600372314453, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.507317304611206, + "logits/rejected": -0.5006474852561951, + "logps/chosen": -385.92010498046875, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -688.0997314453125, + "loss": 1.0973, + "margin_dpo/margin_mean": 272.89990234375, + "margin_dpo/margin_std": 359.01385498046875, + "step": 404 + }, + { + "KL/chosen_KL_mean": -384.9747619628906, + "KL/mean": -494.912841796875, + "KL/rejected_KL_mean": -604.8509521484375, + "KL/std": 318.9220275878906, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.0014201938174664974, + "fcm_dpo/delta": -0.021565556526184082, + "fcm_dpo/margin": 219.87619018554688, + "fcm_dpo/q_t": 0.428875595331192, + "grad_norm": 36.287532806396484, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.5050703287124634, + "logits/rejected": -0.49715399742126465, + "logps/chosen": -450.1376647949219, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -692.0377197265625, + "loss": 1.1772, + "margin_dpo/margin_mean": 219.87620544433594, + "margin_dpo/margin_std": 397.5731201171875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -351.892822265625, + "KL/mean": -498.7806396484375, + "KL/rejected_KL_mean": -645.66845703125, + "KL/std": 319.33978271484375, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.001418529311195016, + "fcm_dpo/delta": -0.020249515771865845, + "fcm_dpo/margin": 293.77557373046875, + "fcm_dpo/q_t": 0.40351957082748413, + "grad_norm": 30.20380401611328, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.5843180418014526, + "logits/rejected": -0.6021308898925781, + "logps/chosen": -401.6336669921875, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -737.7470703125, + "loss": 1.0723, + "margin_dpo/margin_mean": 293.7756042480469, + "margin_dpo/margin_std": 351.3497314453125, + "step": 406 + }, + { + "KL/chosen_KL_mean": -372.94854736328125, + "KL/mean": -463.33843994140625, + "KL/rejected_KL_mean": -553.728271484375, + "KL/std": 250.21084594726562, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.001404798123985529, + "fcm_dpo/delta": 0.025703012943267822, + "fcm_dpo/margin": 180.77969360351562, + "fcm_dpo/q_t": 0.43973639607429504, + "grad_norm": 25.703264236450195, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.5915988683700562, + "logits/rejected": -0.5621857643127441, + "logps/chosen": -429.2792663574219, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -631.2403564453125, + "loss": 1.1989, + "margin_dpo/margin_mean": 180.77969360351562, + "margin_dpo/margin_std": 320.1576843261719, + "step": 407 + }, + { + "KL/chosen_KL_mean": -376.4541015625, + "KL/mean": -478.58209228515625, + "KL/rejected_KL_mean": -580.7101440429688, + "KL/std": 243.93844604492188, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.001487904810346663, + "fcm_dpo/delta": 0.10533779114484787, + "fcm_dpo/margin": 204.25601196289062, + "fcm_dpo/q_t": 0.42959362268447876, + "grad_norm": 29.449756622314453, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.6153702735900879, + "logits/rejected": -0.5948002338409424, + "logps/chosen": -446.243408203125, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -670.8070678710938, + "loss": 1.1457, + "margin_dpo/margin_mean": 204.25601196289062, + "margin_dpo/margin_std": 274.03179931640625, + "step": 408 + }, + { + "KL/chosen_KL_mean": -375.9293212890625, + "KL/mean": -476.3438415527344, + "KL/rejected_KL_mean": -576.7584228515625, + "KL/std": 271.7243347167969, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0015818243846297264, + "fcm_dpo/delta": 0.08981961011886597, + "fcm_dpo/margin": 200.82901000976562, + "fcm_dpo/q_t": 0.4274054169654846, + "grad_norm": 33.60805892944336, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.5516515970230103, + "logits/rejected": -0.5352581739425659, + "logps/chosen": -443.24676513671875, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -661.6626586914062, + "loss": 1.1465, + "margin_dpo/margin_mean": 200.8290252685547, + "margin_dpo/margin_std": 297.3284606933594, + "step": 409 + }, + { + "KL/chosen_KL_mean": -356.83929443359375, + "KL/mean": -481.22149658203125, + "KL/rejected_KL_mean": -605.6036376953125, + "KL/std": 272.7947998046875, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.001588868093676865, + "fcm_dpo/delta": 0.005292635411024094, + "fcm_dpo/margin": 248.76431274414062, + "fcm_dpo/q_t": 0.40695837140083313, + "grad_norm": 32.11767578125, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.6046394109725952, + "logits/rejected": -0.6092381477355957, + "logps/chosen": -408.3046875, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -688.8026123046875, + "loss": 1.0833, + "margin_dpo/margin_mean": 248.76434326171875, + "margin_dpo/margin_std": 298.48699951171875, + "step": 410 + }, + { + "KL/chosen_KL_mean": -375.1066589355469, + "KL/mean": -496.0987548828125, + "KL/rejected_KL_mean": -617.0908813476562, + "KL/std": 298.98272705078125, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0015720587689429522, + "fcm_dpo/delta": 0.018475636839866638, + "fcm_dpo/margin": 241.98422241210938, + "fcm_dpo/q_t": 0.4141007661819458, + "grad_norm": 30.232168197631836, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.6492786407470703, + "logits/rejected": -0.6617485284805298, + "logps/chosen": -427.4139404296875, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -697.7858276367188, + "loss": 1.1173, + "margin_dpo/margin_mean": 241.98422241210938, + "margin_dpo/margin_std": 347.9234313964844, + "step": 411 + }, + { + "KL/chosen_KL_mean": -387.1575927734375, + "KL/mean": -518.0455932617188, + "KL/rejected_KL_mean": -648.93359375, + "KL/std": 296.2761535644531, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0016023230273276567, + "fcm_dpo/delta": -0.02223604917526245, + "fcm_dpo/margin": 261.7760009765625, + "fcm_dpo/q_t": 0.40295469760894775, + "grad_norm": 32.675567626953125, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.6626579165458679, + "logits/rejected": -0.6987204551696777, + "logps/chosen": -440.30169677734375, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -748.994384765625, + "loss": 1.0822, + "margin_dpo/margin_mean": 261.7760314941406, + "margin_dpo/margin_std": 345.51470947265625, + "step": 412 + }, + { + "KL/chosen_KL_mean": -431.1519775390625, + "KL/mean": -561.16015625, + "KL/rejected_KL_mean": -691.1682739257812, + "KL/std": 306.32244873046875, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0015915387775748968, + "fcm_dpo/delta": -0.01595381088554859, + "fcm_dpo/margin": 260.0163269042969, + "fcm_dpo/q_t": 0.40251970291137695, + "grad_norm": 35.346885681152344, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.5932176113128662, + "logits/rejected": -0.6041978597640991, + "logps/chosen": -492.73394775390625, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -790.6417236328125, + "loss": 1.0834, + "margin_dpo/margin_mean": 260.0163269042969, + "margin_dpo/margin_std": 338.83343505859375, + "step": 413 + }, + { + "KL/chosen_KL_mean": -390.7093505859375, + "KL/mean": -527.671630859375, + "KL/rejected_KL_mean": -664.6339111328125, + "KL/std": 288.1386413574219, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0015833561774343252, + "fcm_dpo/delta": -0.041113872081041336, + "fcm_dpo/margin": 273.92462158203125, + "fcm_dpo/q_t": 0.3984673321247101, + "grad_norm": 44.00105285644531, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.5915371775627136, + "logits/rejected": -0.586266040802002, + "logps/chosen": -437.3408508300781, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -752.2804565429688, + "loss": 1.0688, + "margin_dpo/margin_mean": 273.9245910644531, + "margin_dpo/margin_std": 331.4505920410156, + "step": 414 + }, + { + "KL/chosen_KL_mean": -421.54522705078125, + "KL/mean": -531.4916381835938, + "KL/rejected_KL_mean": -641.43798828125, + "KL/std": 288.2938537597656, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0015668061096221209, + "fcm_dpo/delta": 0.06179225072264671, + "fcm_dpo/margin": 219.89276123046875, + "fcm_dpo/q_t": 0.42079904675483704, + "grad_norm": 35.194698333740234, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.6400187611579895, + "logits/rejected": -0.6376417875289917, + "logps/chosen": -500.16351318359375, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -741.91552734375, + "loss": 1.1305, + "margin_dpo/margin_mean": 219.89276123046875, + "margin_dpo/margin_std": 315.3600769042969, + "step": 415 + }, + { + "KL/chosen_KL_mean": -390.89117431640625, + "KL/mean": -558.92724609375, + "KL/rejected_KL_mean": -726.9632568359375, + "KL/std": 327.833251953125, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.001516662654466927, + "fcm_dpo/delta": -0.12909024953842163, + "fcm_dpo/margin": 336.0720520019531, + "fcm_dpo/q_t": 0.3783823847770691, + "grad_norm": 39.02359390258789, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.6565449237823486, + "logits/rejected": -0.6699833869934082, + "logps/chosen": -449.1703186035156, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -817.531982421875, + "loss": 0.9948, + "margin_dpo/margin_mean": 336.07208251953125, + "margin_dpo/margin_std": 317.673095703125, + "step": 416 + }, + { + "KL/chosen_KL_mean": -381.16021728515625, + "KL/mean": -520.4452514648438, + "KL/rejected_KL_mean": -659.730224609375, + "KL/std": 293.77618408203125, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.0014753260184079409, + "fcm_dpo/delta": -0.012566627934575081, + "fcm_dpo/margin": 278.57000732421875, + "fcm_dpo/q_t": 0.402764230966568, + "grad_norm": 26.6540584564209, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.6726227402687073, + "logits/rejected": -0.6600502729415894, + "logps/chosen": -431.35894775390625, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -727.882080078125, + "loss": 1.0607, + "margin_dpo/margin_mean": 278.57000732421875, + "margin_dpo/margin_std": 290.6696472167969, + "step": 417 + }, + { + "KL/chosen_KL_mean": -412.9525146484375, + "KL/mean": -556.04296875, + "KL/rejected_KL_mean": -699.133544921875, + "KL/std": 331.10821533203125, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0014832296874374151, + "fcm_dpo/delta": -0.031469203531742096, + "fcm_dpo/margin": 286.1809387207031, + "fcm_dpo/q_t": 0.4036760926246643, + "grad_norm": 25.186491012573242, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.6861658692359924, + "logits/rejected": -0.6701766848564148, + "logps/chosen": -494.9309997558594, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -803.824951171875, + "loss": 1.0896, + "margin_dpo/margin_mean": 286.1809387207031, + "margin_dpo/margin_std": 387.8187255859375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -378.56024169921875, + "KL/mean": -533.28857421875, + "KL/rejected_KL_mean": -688.0169677734375, + "KL/std": 289.4898376464844, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0014121748972684145, + "fcm_dpo/delta": -0.04262404143810272, + "fcm_dpo/margin": 309.4566650390625, + "fcm_dpo/q_t": 0.3962531089782715, + "grad_norm": 26.683582305908203, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.6140732169151306, + "logits/rejected": -0.6158535480499268, + "logps/chosen": -431.5089111328125, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -779.6000366210938, + "loss": 1.0496, + "margin_dpo/margin_mean": 309.4566650390625, + "margin_dpo/margin_std": 333.7470703125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -458.7154541015625, + "KL/mean": -558.8417358398438, + "KL/rejected_KL_mean": -658.968017578125, + "KL/std": 307.5548095703125, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0014824032550677657, + "fcm_dpo/delta": 0.11253461241722107, + "fcm_dpo/margin": 200.25253295898438, + "fcm_dpo/q_t": 0.43468111753463745, + "grad_norm": 54.70574951171875, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.6774864196777344, + "logits/rejected": -0.6483875513076782, + "logps/chosen": -536.4853515625, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -728.287841796875, + "loss": 1.1955, + "margin_dpo/margin_mean": 200.25253295898438, + "margin_dpo/margin_std": 404.9305725097656, + "step": 420 + }, + { + "KL/chosen_KL_mean": -383.32489013671875, + "KL/mean": -492.3797607421875, + "KL/rejected_KL_mean": -601.4345703125, + "KL/std": 300.0869140625, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0015643943333998322, + "fcm_dpo/delta": 0.0634913370013237, + "fcm_dpo/margin": 218.10971069335938, + "fcm_dpo/q_t": 0.41989845037460327, + "grad_norm": 25.23602867126465, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.6363915205001831, + "logits/rejected": -0.6229156255722046, + "logps/chosen": -437.09075927734375, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -690.716064453125, + "loss": 1.1373, + "margin_dpo/margin_mean": 218.1096954345703, + "margin_dpo/margin_std": 319.56549072265625, + "step": 421 + }, + { + "KL/chosen_KL_mean": -437.31927490234375, + "KL/mean": -563.151611328125, + "KL/rejected_KL_mean": -688.9840087890625, + "KL/std": 303.831298828125, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.00156421959400177, + "fcm_dpo/delta": 0.007191924378275871, + "fcm_dpo/margin": 251.66473388671875, + "fcm_dpo/q_t": 0.4089781641960144, + "grad_norm": 33.02323913574219, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.6678431034088135, + "logits/rejected": -0.6727738976478577, + "logps/chosen": -505.9530029296875, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -776.8475341796875, + "loss": 1.1039, + "margin_dpo/margin_mean": 251.66473388671875, + "margin_dpo/margin_std": 358.28582763671875, + "step": 422 + }, + { + "KL/chosen_KL_mean": -406.384033203125, + "KL/mean": -545.9063720703125, + "KL/rejected_KL_mean": -685.4285888671875, + "KL/std": 296.6026916503906, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0015398156829178333, + "fcm_dpo/delta": -0.034179866313934326, + "fcm_dpo/margin": 279.0445861816406, + "fcm_dpo/q_t": 0.3996448516845703, + "grad_norm": 27.664220809936523, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.6528114080429077, + "logits/rejected": -0.6629751920700073, + "logps/chosen": -461.37432861328125, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -771.7351684570312, + "loss": 1.0612, + "margin_dpo/margin_mean": 279.0445861816406, + "margin_dpo/margin_std": 327.45111083984375, + "step": 423 + }, + { + "KL/chosen_KL_mean": -372.25732421875, + "KL/mean": -500.0310363769531, + "KL/rejected_KL_mean": -627.8046875, + "KL/std": 287.692626953125, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0015238735359162092, + "fcm_dpo/delta": 0.01106809452176094, + "fcm_dpo/margin": 255.5474090576172, + "fcm_dpo/q_t": 0.4116082787513733, + "grad_norm": 33.72771072387695, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.6079432368278503, + "logits/rejected": -0.5840111970901489, + "logps/chosen": -428.269287109375, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -694.28369140625, + "loss": 1.1039, + "margin_dpo/margin_mean": 255.54739379882812, + "margin_dpo/margin_std": 356.714599609375, + "step": 424 + }, + { + "KL/chosen_KL_mean": -409.95977783203125, + "KL/mean": -505.3192138671875, + "KL/rejected_KL_mean": -600.6786499023438, + "KL/std": 276.4864501953125, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.001628840109333396, + "fcm_dpo/delta": 0.09687715768814087, + "fcm_dpo/margin": 190.71890258789062, + "fcm_dpo/q_t": 0.42763423919677734, + "grad_norm": 31.941625595092773, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.6238282918930054, + "logits/rejected": -0.6239417195320129, + "logps/chosen": -456.82879638671875, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -696.6041259765625, + "loss": 1.1747, + "margin_dpo/margin_mean": 190.71890258789062, + "margin_dpo/margin_std": 343.16583251953125, + "step": 425 + }, + { + "KL/chosen_KL_mean": -378.5176086425781, + "KL/mean": -502.3695068359375, + "KL/rejected_KL_mean": -626.2213745117188, + "KL/std": 277.93817138671875, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0016521223587915301, + "fcm_dpo/delta": -0.011752713471651077, + "fcm_dpo/margin": 247.70379638671875, + "fcm_dpo/q_t": 0.4045184850692749, + "grad_norm": 30.411544799804688, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.6232934594154358, + "logits/rejected": -0.5914437174797058, + "logps/chosen": -455.10113525390625, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -707.4879760742188, + "loss": 1.0797, + "margin_dpo/margin_mean": 247.70379638671875, + "margin_dpo/margin_std": 300.90093994140625, + "step": 426 + }, + { + "KL/chosen_KL_mean": -344.63330078125, + "KL/mean": -438.0864562988281, + "KL/rejected_KL_mean": -531.5396118164062, + "KL/std": 241.767822265625, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0016895283479243517, + "fcm_dpo/delta": 0.09324996173381805, + "fcm_dpo/margin": 186.90626525878906, + "fcm_dpo/q_t": 0.42818641662597656, + "grad_norm": 25.489910125732422, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.6217249631881714, + "logits/rejected": -0.6024937629699707, + "logps/chosen": -409.4871826171875, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -610.1055908203125, + "loss": 1.1521, + "margin_dpo/margin_mean": 186.90625, + "margin_dpo/margin_std": 288.1886291503906, + "step": 427 + }, + { + "KL/chosen_KL_mean": -416.2967529296875, + "KL/mean": -548.1687622070312, + "KL/rejected_KL_mean": -680.040771484375, + "KL/std": 312.06298828125, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.0016638458473607898, + "fcm_dpo/delta": -0.046425387263298035, + "fcm_dpo/margin": 263.7440185546875, + "fcm_dpo/q_t": 0.3996710777282715, + "grad_norm": 27.545879364013672, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.5870815515518188, + "logits/rejected": -0.5899187922477722, + "logps/chosen": -478.93341064453125, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -783.3226318359375, + "loss": 1.088, + "margin_dpo/margin_mean": 263.7440185546875, + "margin_dpo/margin_std": 379.46099853515625, + "step": 428 + }, + { + "KL/chosen_KL_mean": -419.58001708984375, + "KL/mean": -521.4755249023438, + "KL/rejected_KL_mean": -623.37109375, + "KL/std": 280.9124450683594, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0016521899960935116, + "fcm_dpo/delta": -0.04059625789523125, + "fcm_dpo/margin": 203.791015625, + "fcm_dpo/q_t": 0.4223303198814392, + "grad_norm": 28.567970275878906, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.5903991460800171, + "logits/rejected": -0.5673129558563232, + "logps/chosen": -500.81402587890625, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -715.166015625, + "loss": 1.1631, + "margin_dpo/margin_mean": 203.791015625, + "margin_dpo/margin_std": 347.600830078125, + "step": 429 + }, + { + "KL/chosen_KL_mean": -344.3188171386719, + "KL/mean": -446.1313171386719, + "KL/rejected_KL_mean": -547.94384765625, + "KL/std": 255.993896484375, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0016840994358062744, + "fcm_dpo/delta": 0.06332562863826752, + "fcm_dpo/margin": 203.62501525878906, + "fcm_dpo/q_t": 0.42141231894493103, + "grad_norm": 32.23469161987305, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.6090602278709412, + "logits/rejected": -0.6085931062698364, + "logps/chosen": -405.2391357421875, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -652.3666381835938, + "loss": 1.1264, + "margin_dpo/margin_mean": 203.625, + "margin_dpo/margin_std": 284.68988037109375, + "step": 430 + }, + { + "KL/chosen_KL_mean": -309.070556640625, + "KL/mean": -460.13092041015625, + "KL/rejected_KL_mean": -611.1912841796875, + "KL/std": 277.47564697265625, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0016117225168272853, + "fcm_dpo/delta": -0.10261335223913193, + "fcm_dpo/margin": 302.1207275390625, + "fcm_dpo/q_t": 0.3841360807418823, + "grad_norm": 24.982919692993164, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.586715817451477, + "logits/rejected": -0.5883519649505615, + "logps/chosen": -366.4193115234375, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -704.031494140625, + "loss": 1.0155, + "margin_dpo/margin_mean": 302.1206970214844, + "margin_dpo/margin_std": 302.3026428222656, + "step": 431 + }, + { + "KL/chosen_KL_mean": -309.99102783203125, + "KL/mean": -454.25384521484375, + "KL/rejected_KL_mean": -598.5166625976562, + "KL/std": 268.39483642578125, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.001578683266416192, + "fcm_dpo/delta": -0.0655520111322403, + "fcm_dpo/margin": 288.52557373046875, + "fcm_dpo/q_t": 0.39216262102127075, + "grad_norm": 31.187776565551758, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.5495719313621521, + "logits/rejected": -0.540544867515564, + "logps/chosen": -381.06585693359375, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -697.09619140625, + "loss": 1.0312, + "margin_dpo/margin_mean": 288.52557373046875, + "margin_dpo/margin_std": 276.2086486816406, + "step": 432 + }, + { + "KL/chosen_KL_mean": -398.5560302734375, + "KL/mean": -497.3359375, + "KL/rejected_KL_mean": -596.1158447265625, + "KL/std": 270.40673828125, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0016037577297538519, + "fcm_dpo/delta": 0.09126869589090347, + "fcm_dpo/margin": 197.55984497070312, + "fcm_dpo/q_t": 0.4259822368621826, + "grad_norm": 30.545278549194336, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.5997741222381592, + "logits/rejected": -0.5896806120872498, + "logps/chosen": -456.8292236328125, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -692.0667724609375, + "loss": 1.1586, + "margin_dpo/margin_mean": 197.55984497070312, + "margin_dpo/margin_std": 322.60174560546875, + "step": 433 + }, + { + "KL/chosen_KL_mean": -345.7264404296875, + "KL/mean": -466.3848876953125, + "KL/rejected_KL_mean": -587.0432739257812, + "KL/std": 276.73077392578125, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0016135682817548513, + "fcm_dpo/delta": 0.011616135016083717, + "fcm_dpo/margin": 241.31678771972656, + "fcm_dpo/q_t": 0.41396236419677734, + "grad_norm": 23.370208740234375, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.603460431098938, + "logits/rejected": -0.6028808355331421, + "logps/chosen": -407.7001647949219, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -665.5418701171875, + "loss": 1.1033, + "margin_dpo/margin_mean": 241.31678771972656, + "margin_dpo/margin_std": 348.450927734375, + "step": 434 + }, + { + "KL/chosen_KL_mean": -324.51959228515625, + "KL/mean": -454.09930419921875, + "KL/rejected_KL_mean": -583.678955078125, + "KL/std": 277.306396484375, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0016115298494696617, + "fcm_dpo/delta": -0.020269781351089478, + "fcm_dpo/margin": 259.1593933105469, + "fcm_dpo/q_t": 0.40234375, + "grad_norm": 28.945114135742188, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.6465270519256592, + "logits/rejected": -0.638401210308075, + "logps/chosen": -376.02166748046875, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -671.245849609375, + "loss": 1.0735, + "margin_dpo/margin_mean": 259.15936279296875, + "margin_dpo/margin_std": 309.1746826171875, + "step": 435 + }, + { + "KL/chosen_KL_mean": -357.98419189453125, + "KL/mean": -471.7822570800781, + "KL/rejected_KL_mean": -585.580322265625, + "KL/std": 251.277099609375, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0016217220108956099, + "fcm_dpo/delta": 0.03454357385635376, + "fcm_dpo/margin": 227.59609985351562, + "fcm_dpo/q_t": 0.41459012031555176, + "grad_norm": 34.471099853515625, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.5865396857261658, + "logits/rejected": -0.5578336715698242, + "logps/chosen": -429.387939453125, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -668.30810546875, + "loss": 1.1069, + "margin_dpo/margin_mean": 227.59609985351562, + "margin_dpo/margin_std": 298.0869445800781, + "step": 436 + }, + { + "KL/chosen_KL_mean": -374.7752990722656, + "KL/mean": -489.54827880859375, + "KL/rejected_KL_mean": -604.3211669921875, + "KL/std": 254.74867248535156, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0016534898895770311, + "fcm_dpo/delta": 0.023033857345581055, + "fcm_dpo/margin": 229.5458984375, + "fcm_dpo/q_t": 0.41207918524742126, + "grad_norm": 23.10723876953125, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.6237749457359314, + "logits/rejected": -0.6015676259994507, + "logps/chosen": -439.5195617675781, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -686.36474609375, + "loss": 1.1045, + "margin_dpo/margin_mean": 229.5458984375, + "margin_dpo/margin_std": 309.5416564941406, + "step": 437 + }, + { + "KL/chosen_KL_mean": -387.9025573730469, + "KL/mean": -520.5615844726562, + "KL/rejected_KL_mean": -653.2205810546875, + "KL/std": 285.2457580566406, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0016199819510802627, + "fcm_dpo/delta": -0.035875964909791946, + "fcm_dpo/margin": 265.31805419921875, + "fcm_dpo/q_t": 0.39900100231170654, + "grad_norm": 36.14478302001953, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.6407413482666016, + "logits/rejected": -0.6248997449874878, + "logps/chosen": -446.9212341308594, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -736.2974243164062, + "loss": 1.0624, + "margin_dpo/margin_mean": 265.31805419921875, + "margin_dpo/margin_std": 309.3646240234375, + "step": 438 + }, + { + "KL/chosen_KL_mean": -414.10455322265625, + "KL/mean": -524.5791625976562, + "KL/rejected_KL_mean": -635.0537719726562, + "KL/std": 292.762451171875, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0015559683088213205, + "fcm_dpo/delta": -0.07906072586774826, + "fcm_dpo/margin": 220.94924926757812, + "fcm_dpo/q_t": 0.4191586375236511, + "grad_norm": 29.60694122314453, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.6627211570739746, + "logits/rejected": -0.6662489175796509, + "logps/chosen": -467.88861083984375, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -719.0392456054688, + "loss": 1.144, + "margin_dpo/margin_mean": 220.94924926757812, + "margin_dpo/margin_std": 337.2562561035156, + "step": 439 + }, + { + "KL/chosen_KL_mean": -445.576171875, + "KL/mean": -578.6038818359375, + "KL/rejected_KL_mean": -711.6316528320312, + "KL/std": 358.3684997558594, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.001555685419589281, + "fcm_dpo/delta": -0.0160941444337368, + "fcm_dpo/margin": 266.055419921875, + "fcm_dpo/q_t": 0.40817397832870483, + "grad_norm": 30.581890106201172, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.7077510356903076, + "logits/rejected": -0.7070698142051697, + "logps/chosen": -524.1428833007812, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -808.12939453125, + "loss": 1.0888, + "margin_dpo/margin_mean": 266.055419921875, + "margin_dpo/margin_std": 373.1027526855469, + "step": 440 + }, + { + "KL/chosen_KL_mean": -510.38885498046875, + "KL/mean": -641.4425048828125, + "KL/rejected_KL_mean": -772.4962158203125, + "KL/std": 380.78594970703125, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0015536651480942965, + "fcm_dpo/delta": -0.00916876271367073, + "fcm_dpo/margin": 262.1072998046875, + "fcm_dpo/q_t": 0.4120785593986511, + "grad_norm": 41.169288635253906, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.7156891822814941, + "logits/rejected": -0.7044844031333923, + "logps/chosen": -571.2132568359375, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -868.9669799804688, + "loss": 1.139, + "margin_dpo/margin_mean": 262.1072998046875, + "margin_dpo/margin_std": 460.61407470703125, + "step": 441 + }, + { + "KL/chosen_KL_mean": -439.59686279296875, + "KL/mean": -596.8812255859375, + "KL/rejected_KL_mean": -754.1656494140625, + "KL/std": 365.017333984375, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0015060793375596404, + "fcm_dpo/delta": -0.08716575056314468, + "fcm_dpo/margin": 314.5688171386719, + "fcm_dpo/q_t": 0.393669456243515, + "grad_norm": 30.335174560546875, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.7000541687011719, + "logits/rejected": -0.6957566738128662, + "logps/chosen": -486.6080627441406, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -830.7049560546875, + "loss": 1.0651, + "margin_dpo/margin_mean": 314.56878662109375, + "margin_dpo/margin_std": 424.580322265625, + "step": 442 + }, + { + "KL/chosen_KL_mean": -508.39227294921875, + "KL/mean": -604.338623046875, + "KL/rejected_KL_mean": -700.284912109375, + "KL/std": 363.72418212890625, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0014648688957095146, + "fcm_dpo/delta": 0.011874865740537643, + "fcm_dpo/margin": 191.89268493652344, + "fcm_dpo/q_t": 0.43707412481307983, + "grad_norm": 34.13652801513672, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.6903680562973022, + "logits/rejected": -0.6664457321166992, + "logps/chosen": -579.665283203125, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -786.9649047851562, + "loss": 1.2266, + "margin_dpo/margin_mean": 191.89266967773438, + "margin_dpo/margin_std": 445.571533203125, + "step": 443 + }, + { + "KL/chosen_KL_mean": -517.1072998046875, + "KL/mean": -673.7557373046875, + "KL/rejected_KL_mean": -830.4041748046875, + "KL/std": 388.1787414550781, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.001416308106854558, + "fcm_dpo/delta": -0.05186297744512558, + "fcm_dpo/margin": 313.2969055175781, + "fcm_dpo/q_t": 0.3994150161743164, + "grad_norm": 32.83633804321289, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.6624064445495605, + "logits/rejected": -0.6699991226196289, + "logps/chosen": -574.3209838867188, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -927.6590576171875, + "loss": 1.076, + "margin_dpo/margin_mean": 313.296875, + "margin_dpo/margin_std": 432.3821716308594, + "step": 444 + }, + { + "KL/chosen_KL_mean": -459.6697998046875, + "KL/mean": -605.974365234375, + "KL/rejected_KL_mean": -752.2788696289062, + "KL/std": 305.039306640625, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0014182128943502903, + "fcm_dpo/delta": -0.017287882044911385, + "fcm_dpo/margin": 292.60906982421875, + "fcm_dpo/q_t": 0.4050604999065399, + "grad_norm": 31.51605224609375, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.7042147517204285, + "logits/rejected": -0.7021005153656006, + "logps/chosen": -526.9696044921875, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -844.9615478515625, + "loss": 1.0852, + "margin_dpo/margin_mean": 292.60906982421875, + "margin_dpo/margin_std": 383.3020935058594, + "step": 445 + }, + { + "KL/chosen_KL_mean": -408.97991943359375, + "KL/mean": -567.7203369140625, + "KL/rejected_KL_mean": -726.4608154296875, + "KL/std": 327.78521728515625, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.001397131709381938, + "fcm_dpo/delta": -0.051334552466869354, + "fcm_dpo/margin": 317.4808654785156, + "fcm_dpo/q_t": 0.39486032724380493, + "grad_norm": 28.071739196777344, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.7243236899375916, + "logits/rejected": -0.7423403263092041, + "logps/chosen": -468.0783996582031, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -827.7249755859375, + "loss": 1.0472, + "margin_dpo/margin_mean": 317.4808654785156, + "margin_dpo/margin_std": 337.61065673828125, + "step": 446 + }, + { + "KL/chosen_KL_mean": -435.85357666015625, + "KL/mean": -598.9805297851562, + "KL/rejected_KL_mean": -762.107421875, + "KL/std": 379.1376647949219, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0013276372337713838, + "fcm_dpo/delta": -0.03901593014597893, + "fcm_dpo/margin": 326.2538757324219, + "fcm_dpo/q_t": 0.40134865045547485, + "grad_norm": 27.509519577026367, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.7886593341827393, + "logits/rejected": -0.8069747090339661, + "logps/chosen": -491.92889404296875, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -860.80224609375, + "loss": 1.0704, + "margin_dpo/margin_mean": 326.2538757324219, + "margin_dpo/margin_std": 426.79656982421875, + "step": 447 + }, + { + "KL/chosen_KL_mean": -475.12237548828125, + "KL/mean": -624.1657104492188, + "KL/rejected_KL_mean": -773.2091064453125, + "KL/std": 335.6260070800781, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.001342701492831111, + "fcm_dpo/delta": -0.0008368827402591705, + "fcm_dpo/margin": 298.086669921875, + "fcm_dpo/q_t": 0.4070809781551361, + "grad_norm": 34.63826370239258, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.6973883509635925, + "logits/rejected": -0.7095668315887451, + "logps/chosen": -535.126220703125, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -875.4737548828125, + "loss": 1.0878, + "margin_dpo/margin_mean": 298.0867004394531, + "margin_dpo/margin_std": 381.0465087890625, + "step": 448 + }, + { + "KL/chosen_KL_mean": -471.00701904296875, + "KL/mean": -627.846435546875, + "KL/rejected_KL_mean": -784.6859130859375, + "KL/std": 405.4683837890625, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.0013168163131922483, + "fcm_dpo/delta": -0.015059142373502254, + "fcm_dpo/margin": 313.6789245605469, + "fcm_dpo/q_t": 0.4089391231536865, + "grad_norm": 27.030296325683594, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.7177699208259583, + "logits/rejected": -0.7121981382369995, + "logps/chosen": -538.5336303710938, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -873.2828369140625, + "loss": 1.1037, + "margin_dpo/margin_mean": 313.678955078125, + "margin_dpo/margin_std": 484.75225830078125, + "step": 449 + }, + { + "KL/chosen_KL_mean": -370.7376708984375, + "KL/mean": -544.8502197265625, + "KL/rejected_KL_mean": -718.9627075195312, + "KL/std": 351.2831726074219, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0012921716552227736, + "fcm_dpo/delta": -0.057655178010463715, + "fcm_dpo/margin": 348.22503662109375, + "fcm_dpo/q_t": 0.39258208870887756, + "grad_norm": 42.00742721557617, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.7456691861152649, + "logits/rejected": -0.7616350650787354, + "logps/chosen": -427.8457946777344, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -821.7176513671875, + "loss": 1.0323, + "margin_dpo/margin_mean": 348.22503662109375, + "margin_dpo/margin_std": 341.9737548828125, + "step": 450 + }, + { + "KL/chosen_KL_mean": -520.0347900390625, + "KL/mean": -640.217529296875, + "KL/rejected_KL_mean": -760.4002685546875, + "KL/std": 412.00030517578125, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.001334954984486103, + "fcm_dpo/delta": 0.0862482488155365, + "fcm_dpo/margin": 240.3655242919922, + "fcm_dpo/q_t": 0.424540638923645, + "grad_norm": 34.657989501953125, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.8403540253639221, + "logits/rejected": -0.8179661631584167, + "logps/chosen": -578.503662109375, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -833.3297119140625, + "loss": 1.1901, + "margin_dpo/margin_mean": 240.3655242919922, + "margin_dpo/margin_std": 490.3316650390625, + "step": 451 + }, + { + "KL/chosen_KL_mean": -368.7062683105469, + "KL/mean": -510.607421875, + "KL/rejected_KL_mean": -652.508544921875, + "KL/std": 299.5830078125, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.001348359975963831, + "fcm_dpo/delta": 0.019563382491469383, + "fcm_dpo/margin": 283.80230712890625, + "fcm_dpo/q_t": 0.4096114933490753, + "grad_norm": 28.84398078918457, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.7127253413200378, + "logits/rejected": -0.7305305600166321, + "logps/chosen": -419.0968017578125, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -750.2799682617188, + "loss": 1.0812, + "margin_dpo/margin_mean": 283.80230712890625, + "margin_dpo/margin_std": 307.7044982910156, + "step": 452 + }, + { + "KL/chosen_KL_mean": -422.7850341796875, + "KL/mean": -564.8392333984375, + "KL/rejected_KL_mean": -706.8934326171875, + "KL/std": 321.3519287109375, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0013556303456425667, + "fcm_dpo/delta": 0.016425350680947304, + "fcm_dpo/margin": 284.10845947265625, + "fcm_dpo/q_t": 0.41288626194000244, + "grad_norm": 33.13711166381836, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.7530020475387573, + "logits/rejected": -0.739249050617218, + "logps/chosen": -480.4998779296875, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -789.1008911132812, + "loss": 1.1024, + "margin_dpo/margin_mean": 284.1084289550781, + "margin_dpo/margin_std": 391.07159423828125, + "step": 453 + }, + { + "KL/chosen_KL_mean": -504.71710205078125, + "KL/mean": -667.727294921875, + "KL/rejected_KL_mean": -830.7374877929688, + "KL/std": 384.3804931640625, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.001335537526756525, + "fcm_dpo/delta": -0.04147890955209732, + "fcm_dpo/margin": 326.0203857421875, + "fcm_dpo/q_t": 0.3995750844478607, + "grad_norm": 28.278629302978516, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.8023015260696411, + "logits/rejected": -0.794007420539856, + "logps/chosen": -565.6627197265625, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -915.688232421875, + "loss": 1.0673, + "margin_dpo/margin_mean": 326.0203857421875, + "margin_dpo/margin_std": 417.18914794921875, + "step": 454 + }, + { + "KL/chosen_KL_mean": -435.348388671875, + "KL/mean": -626.329345703125, + "KL/rejected_KL_mean": -817.3104248046875, + "KL/std": 382.108154296875, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.001296892762184143, + "fcm_dpo/delta": -0.11234508454799652, + "fcm_dpo/margin": 381.9620056152344, + "fcm_dpo/q_t": 0.3853432834148407, + "grad_norm": 30.19273567199707, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.7646974325180054, + "logits/rejected": -0.7885594964027405, + "logps/chosen": -480.235107421875, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -932.6118774414062, + "loss": 1.0301, + "margin_dpo/margin_mean": 381.9620361328125, + "margin_dpo/margin_std": 438.4046630859375, + "step": 455 + }, + { + "KL/chosen_KL_mean": -465.11260986328125, + "KL/mean": -649.96923828125, + "KL/rejected_KL_mean": -834.825927734375, + "KL/std": 396.87213134765625, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.00122337625361979, + "fcm_dpo/delta": -0.060386668890714645, + "fcm_dpo/margin": 369.7133483886719, + "fcm_dpo/q_t": 0.39421674609184265, + "grad_norm": 30.606672286987305, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.7834040522575378, + "logits/rejected": -0.8101434707641602, + "logps/chosen": -522.1493530273438, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -940.0437622070312, + "loss": 1.0337, + "margin_dpo/margin_mean": 369.7133483886719, + "margin_dpo/margin_std": 381.18206787109375, + "step": 456 + }, + { + "KL/chosen_KL_mean": -451.7387390136719, + "KL/mean": -638.734130859375, + "KL/rejected_KL_mean": -825.7294921875, + "KL/std": 391.43475341796875, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0011723204515874386, + "fcm_dpo/delta": -0.045070793479681015, + "fcm_dpo/margin": 373.99072265625, + "fcm_dpo/q_t": 0.39582881331443787, + "grad_norm": 38.951271057128906, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.7769747376441956, + "logits/rejected": -0.7797998189926147, + "logps/chosen": -505.98126220703125, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -910.8390502929688, + "loss": 1.0435, + "margin_dpo/margin_mean": 373.99072265625, + "margin_dpo/margin_std": 382.50921630859375, + "step": 457 + }, + { + "KL/chosen_KL_mean": -494.71368408203125, + "KL/mean": -650.247314453125, + "KL/rejected_KL_mean": -805.7808837890625, + "KL/std": 355.7939453125, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.0011750641278922558, + "fcm_dpo/delta": 0.03765374794602394, + "fcm_dpo/margin": 311.06719970703125, + "fcm_dpo/q_t": 0.41716766357421875, + "grad_norm": 28.636030197143555, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.8248865604400635, + "logits/rejected": -0.8205466866493225, + "logps/chosen": -550.12255859375, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -903.464111328125, + "loss": 1.1126, + "margin_dpo/margin_mean": 311.0671691894531, + "margin_dpo/margin_std": 415.38037109375, + "step": 458 + }, + { + "KL/chosen_KL_mean": -517.1803588867188, + "KL/mean": -702.5677490234375, + "KL/rejected_KL_mean": -887.9550170898438, + "KL/std": 419.85528564453125, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0011653017718344927, + "fcm_dpo/delta": -0.03842644393444061, + "fcm_dpo/margin": 370.7746887207031, + "fcm_dpo/q_t": 0.40038570761680603, + "grad_norm": 28.49128532409668, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.8735007047653198, + "logits/rejected": -0.8952823281288147, + "logps/chosen": -563.7378540039062, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -974.12353515625, + "loss": 1.0718, + "margin_dpo/margin_mean": 370.77471923828125, + "margin_dpo/margin_std": 471.01177978515625, + "step": 459 + }, + { + "KL/chosen_KL_mean": -576.1680908203125, + "KL/mean": -776.01708984375, + "KL/rejected_KL_mean": -975.8662109375, + "KL/std": 412.54986572265625, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.001142516965046525, + "fcm_dpo/delta": -0.06553258001804352, + "fcm_dpo/margin": 399.6980895996094, + "fcm_dpo/q_t": 0.3939363360404968, + "grad_norm": 34.0096435546875, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.883228063583374, + "logits/rejected": -0.9167052507400513, + "logps/chosen": -627.802978515625, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -1079.985595703125, + "loss": 1.0416, + "margin_dpo/margin_mean": 399.6980895996094, + "margin_dpo/margin_std": 453.53997802734375, + "step": 460 + }, + { + "KL/chosen_KL_mean": -616.5801391601562, + "KL/mean": -777.030029296875, + "KL/rejected_KL_mean": -937.4798583984375, + "KL/std": 415.2491455078125, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0011540568666532636, + "fcm_dpo/delta": 0.0325024239718914, + "fcm_dpo/margin": 320.8997497558594, + "fcm_dpo/q_t": 0.41613778471946716, + "grad_norm": 25.42841339111328, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.8740191459655762, + "logits/rejected": -0.8918579816818237, + "logps/chosen": -671.7620849609375, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1023.956787109375, + "loss": 1.1142, + "margin_dpo/margin_mean": 320.89971923828125, + "margin_dpo/margin_std": 457.2764892578125, + "step": 461 + }, + { + "KL/chosen_KL_mean": -639.719482421875, + "KL/mean": -777.9473876953125, + "KL/rejected_KL_mean": -916.1752319335938, + "KL/std": 442.97686767578125, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.001190143870189786, + "fcm_dpo/delta": 0.07877114415168762, + "fcm_dpo/margin": 276.4557800292969, + "fcm_dpo/q_t": 0.4280284643173218, + "grad_norm": 31.730234146118164, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.9657743573188782, + "logits/rejected": -0.9539197683334351, + "logps/chosen": -709.6475219726562, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -995.016357421875, + "loss": 1.1637, + "margin_dpo/margin_mean": 276.4557800292969, + "margin_dpo/margin_std": 494.6156311035156, + "step": 462 + }, + { + "KL/chosen_KL_mean": -646.419921875, + "KL/mean": -821.9989013671875, + "KL/rejected_KL_mean": -997.5778198242188, + "KL/std": 444.561767578125, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.0012035011313855648, + "fcm_dpo/delta": -0.026358643546700478, + "fcm_dpo/margin": 351.15789794921875, + "fcm_dpo/q_t": 0.404382586479187, + "grad_norm": 38.75714874267578, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.9487147331237793, + "logits/rejected": -0.9604390859603882, + "logps/chosen": -701.6943359375, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -1086.602783203125, + "loss": 1.0906, + "margin_dpo/margin_mean": 351.1578674316406, + "margin_dpo/margin_std": 495.43133544921875, + "step": 463 + }, + { + "KL/chosen_KL_mean": -627.3611450195312, + "KL/mean": -893.9738159179688, + "KL/rejected_KL_mean": -1160.5865478515625, + "KL/std": 515.2576293945312, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0010559436632320285, + "fcm_dpo/delta": -0.20667815208435059, + "fcm_dpo/margin": 533.225341796875, + "fcm_dpo/q_t": 0.3653695583343506, + "grad_norm": 34.538326263427734, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.9071370363235474, + "logits/rejected": -0.9753853678703308, + "logps/chosen": -678.2734375, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1263.075927734375, + "loss": 0.9713, + "margin_dpo/margin_mean": 533.225341796875, + "margin_dpo/margin_std": 528.76123046875, + "step": 464 + }, + { + "KL/chosen_KL_mean": -670.5797729492188, + "KL/mean": -894.870849609375, + "KL/rejected_KL_mean": -1119.162109375, + "KL/std": 562.9317626953125, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0010279943235218525, + "fcm_dpo/delta": -0.07110614329576492, + "fcm_dpo/margin": 448.582275390625, + "fcm_dpo/q_t": 0.39131027460098267, + "grad_norm": 34.86879348754883, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.8944182991981506, + "logits/rejected": -0.9228054285049438, + "logps/chosen": -730.6966552734375, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -1233.108154296875, + "loss": 1.0672, + "margin_dpo/margin_mean": 448.582275390625, + "margin_dpo/margin_std": 597.0631103515625, + "step": 465 + }, + { + "KL/chosen_KL_mean": -742.7828979492188, + "KL/mean": -927.717529296875, + "KL/rejected_KL_mean": -1112.652099609375, + "KL/std": 520.524658203125, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0010030376724898815, + "fcm_dpo/delta": 0.031469956040382385, + "fcm_dpo/margin": 369.8692626953125, + "fcm_dpo/q_t": 0.4150125980377197, + "grad_norm": 30.259843826293945, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.9544075727462769, + "logits/rejected": -0.9589208364486694, + "logps/chosen": -795.7037963867188, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -1202.967529296875, + "loss": 1.129, + "margin_dpo/margin_mean": 369.8692626953125, + "margin_dpo/margin_std": 566.941162109375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -935.7764282226562, + "KL/mean": -1134.380859375, + "KL/rejected_KL_mean": -1332.985107421875, + "KL/std": 665.6499633789062, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0009985107462853193, + "fcm_dpo/delta": -0.0004987195134162903, + "fcm_dpo/margin": 397.20880126953125, + "fcm_dpo/q_t": 0.41237157583236694, + "grad_norm": 45.825538635253906, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -1.0273045301437378, + "logits/rejected": -1.022709846496582, + "logps/chosen": -1014.4922485351562, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1435.845458984375, + "loss": 1.1737, + "margin_dpo/margin_mean": 397.20880126953125, + "margin_dpo/margin_std": 781.654296875, + "step": 467 + }, + { + "KL/chosen_KL_mean": -723.331787109375, + "KL/mean": -979.1888427734375, + "KL/rejected_KL_mean": -1235.0458984375, + "KL/std": 607.69970703125, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0009727263823151588, + "fcm_dpo/delta": -0.11494794487953186, + "fcm_dpo/margin": 511.7141418457031, + "fcm_dpo/q_t": 0.38931429386138916, + "grad_norm": 31.650243759155273, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.9722326993942261, + "logits/rejected": -0.9880881309509277, + "logps/chosen": -793.2671508789062, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1336.07470703125, + "loss": 1.0455, + "margin_dpo/margin_mean": 511.71417236328125, + "margin_dpo/margin_std": 693.203369140625, + "step": 468 + }, + { + "KL/chosen_KL_mean": -767.3590087890625, + "KL/mean": -1022.5811157226562, + "KL/rejected_KL_mean": -1277.80322265625, + "KL/std": 551.5078125, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0009053861722350121, + "fcm_dpo/delta": -0.07644487172365189, + "fcm_dpo/margin": 510.4442138671875, + "fcm_dpo/q_t": 0.3916725218296051, + "grad_norm": 35.91495895385742, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -1.018198013305664, + "logits/rejected": -1.0455138683319092, + "logps/chosen": -835.4837036132812, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1382.589599609375, + "loss": 1.0474, + "margin_dpo/margin_mean": 510.4442138671875, + "margin_dpo/margin_std": 599.939208984375, + "step": 469 + }, + { + "KL/chosen_KL_mean": -647.519775390625, + "KL/mean": -854.4847412109375, + "KL/rejected_KL_mean": -1061.449462890625, + "KL/std": 541.2998046875, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0009283692925237119, + "fcm_dpo/delta": 0.01684669218957424, + "fcm_dpo/margin": 413.92974853515625, + "fcm_dpo/q_t": 0.4118465185165405, + "grad_norm": 29.18634033203125, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.9446853399276733, + "logits/rejected": -0.9629919528961182, + "logps/chosen": -691.311767578125, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1144.15234375, + "loss": 1.1171, + "margin_dpo/margin_mean": 413.92974853515625, + "margin_dpo/margin_std": 619.9808959960938, + "step": 470 + }, + { + "KL/chosen_KL_mean": -801.2978515625, + "KL/mean": -978.8816528320312, + "KL/rejected_KL_mean": -1156.465576171875, + "KL/std": 578.956298828125, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0009118622401729226, + "fcm_dpo/delta": -0.017206627875566483, + "fcm_dpo/margin": 355.16766357421875, + "fcm_dpo/q_t": 0.42804932594299316, + "grad_norm": 36.21306610107422, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -1.0602209568023682, + "logits/rejected": -1.0760873556137085, + "logps/chosen": -864.6373291015625, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1240.075927734375, + "loss": 1.1698, + "margin_dpo/margin_mean": 355.16766357421875, + "margin_dpo/margin_std": 648.285888671875, + "step": 471 + }, + { + "KL/chosen_KL_mean": -792.2078857421875, + "KL/mean": -1009.8846435546875, + "KL/rejected_KL_mean": -1227.5615234375, + "KL/std": 670.2850341796875, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0009059185977093875, + "fcm_dpo/delta": 0.0058145672082901, + "fcm_dpo/margin": 435.35357666015625, + "fcm_dpo/q_t": 0.4137437641620636, + "grad_norm": 29.12934112548828, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.9907628297805786, + "logits/rejected": -0.9731810092926025, + "logps/chosen": -875.8740234375, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1344.770751953125, + "loss": 1.1319, + "margin_dpo/margin_mean": 435.35357666015625, + "margin_dpo/margin_std": 729.3368530273438, + "step": 472 + }, + { + "KL/chosen_KL_mean": -927.4757080078125, + "KL/mean": -1024.789794921875, + "KL/rejected_KL_mean": -1122.10400390625, + "KL/std": 678.0457763671875, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.000915049051400274, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 194.6282958984375, + "fcm_dpo/q_t": 0.4611557126045227, + "grad_norm": 99.40251922607422, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -1.0223989486694336, + "logits/rejected": -1.001873254776001, + "logps/chosen": -990.97265625, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1203.2506103515625, + "loss": 1.366, + "margin_dpo/margin_mean": 194.6282958984375, + "margin_dpo/margin_std": 902.9514770507812, + "step": 473 + }, + { + "KL/chosen_KL_mean": -738.7838134765625, + "KL/mean": -925.6276245117188, + "KL/rejected_KL_mean": -1112.471435546875, + "KL/std": 601.6588134765625, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.0008969607297331095, + "fcm_dpo/delta": -0.06722978502511978, + "fcm_dpo/margin": 373.6876220703125, + "fcm_dpo/q_t": 0.42289626598358154, + "grad_norm": 31.58320426940918, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.9379159212112427, + "logits/rejected": -0.9463539123535156, + "logps/chosen": -791.395751953125, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1202.5518798828125, + "loss": 1.1779, + "margin_dpo/margin_mean": 373.6876220703125, + "margin_dpo/margin_std": 690.6107788085938, + "step": 474 + }, + { + "KL/chosen_KL_mean": -539.2564697265625, + "KL/mean": -773.5672607421875, + "KL/rejected_KL_mean": -1007.8779907226562, + "KL/std": 467.53741455078125, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0008714465657249093, + "fcm_dpo/delta": -0.009715787135064602, + "fcm_dpo/margin": 468.6214904785156, + "fcm_dpo/q_t": 0.40400421619415283, + "grad_norm": 26.58054542541504, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.9807555079460144, + "logits/rejected": -1.0123473405838013, + "logps/chosen": -581.751708984375, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1097.94091796875, + "loss": 1.0656, + "margin_dpo/margin_mean": 468.6214904785156, + "margin_dpo/margin_std": 508.9897155761719, + "step": 475 + }, + { + "KL/chosen_KL_mean": -649.3043212890625, + "KL/mean": -833.1800537109375, + "KL/rejected_KL_mean": -1017.0557250976562, + "KL/std": 528.4341430664062, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.0009031184017658234, + "fcm_dpo/delta": 0.07541535794734955, + "fcm_dpo/margin": 367.75146484375, + "fcm_dpo/q_t": 0.4247477650642395, + "grad_norm": 48.83927536010742, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.9435632228851318, + "logits/rejected": -0.9667356014251709, + "logps/chosen": -692.2537231445312, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1090.7659912109375, + "loss": 1.1351, + "margin_dpo/margin_mean": 367.75146484375, + "margin_dpo/margin_std": 531.6053466796875, + "step": 476 + }, + { + "KL/chosen_KL_mean": -681.8865966796875, + "KL/mean": -864.00390625, + "KL/rejected_KL_mean": -1046.1212158203125, + "KL/std": 526.7784423828125, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0009477640269324183, + "fcm_dpo/delta": 0.059899330139160156, + "fcm_dpo/margin": 364.23468017578125, + "fcm_dpo/q_t": 0.4214591979980469, + "grad_norm": 27.471927642822266, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -1.0199906826019287, + "logits/rejected": -0.9938687086105347, + "logps/chosen": -752.6591796875, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1122.258544921875, + "loss": 1.1574, + "margin_dpo/margin_mean": 364.23468017578125, + "margin_dpo/margin_std": 642.031494140625, + "step": 477 + }, + { + "KL/chosen_KL_mean": -565.2698974609375, + "KL/mean": -780.1640625, + "KL/rejected_KL_mean": -995.0582275390625, + "KL/std": 473.66259765625, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.000951889727730304, + "fcm_dpo/delta": -0.010887086391448975, + "fcm_dpo/margin": 429.78826904296875, + "fcm_dpo/q_t": 0.4055454730987549, + "grad_norm": 29.83587646484375, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.8782244920730591, + "logits/rejected": -0.904800534248352, + "logps/chosen": -606.71044921875, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1080.420166015625, + "loss": 1.0841, + "margin_dpo/margin_mean": 429.78826904296875, + "margin_dpo/margin_std": 554.5184326171875, + "step": 478 + }, + { + "KL/chosen_KL_mean": -711.8953857421875, + "KL/mean": -915.696533203125, + "KL/rejected_KL_mean": -1119.497802734375, + "KL/std": 590.8914184570312, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0009643921512179077, + "fcm_dpo/delta": 0.004330448806285858, + "fcm_dpo/margin": 407.602294921875, + "fcm_dpo/q_t": 0.4132574498653412, + "grad_norm": 27.756729125976562, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.9402629137039185, + "logits/rejected": -0.9760748147964478, + "logps/chosen": -765.8033447265625, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1214.614013671875, + "loss": 1.1313, + "margin_dpo/margin_mean": 407.6022644042969, + "margin_dpo/margin_std": 678.027099609375, + "step": 479 + }, + { + "KL/chosen_KL_mean": -864.3367919921875, + "KL/mean": -1028.92626953125, + "KL/rejected_KL_mean": -1193.515625, + "KL/std": 552.187744140625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0009353804634884, + "fcm_dpo/delta": -0.03051244653761387, + "fcm_dpo/margin": 329.1788024902344, + "fcm_dpo/q_t": 0.4297601580619812, + "grad_norm": 46.92842483520508, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -1.0446572303771973, + "logits/rejected": -1.0497450828552246, + "logps/chosen": -923.01953125, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1276.4481201171875, + "loss": 1.1832, + "margin_dpo/margin_mean": 329.1788024902344, + "margin_dpo/margin_std": 594.969482421875, + "step": 480 + }, + { + "KL/chosen_KL_mean": -729.44921875, + "KL/mean": -974.4056396484375, + "KL/rejected_KL_mean": -1219.362060546875, + "KL/std": 588.7672119140625, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0009089148370549083, + "fcm_dpo/delta": -0.052177466452121735, + "fcm_dpo/margin": 489.91290283203125, + "fcm_dpo/q_t": 0.39790213108062744, + "grad_norm": 30.010175704956055, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -1.0348255634307861, + "logits/rejected": -1.066978096961975, + "logps/chosen": -784.4134521484375, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1311.782470703125, + "loss": 1.0667, + "margin_dpo/margin_mean": 489.91290283203125, + "margin_dpo/margin_std": 635.6770629882812, + "step": 481 + }, + { + "KL/chosen_KL_mean": -792.1005859375, + "KL/mean": -913.0858154296875, + "KL/rejected_KL_mean": -1034.0711669921875, + "KL/std": 590.38037109375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0009204308735206723, + "fcm_dpo/delta": 0.0803341418504715, + "fcm_dpo/margin": 241.9705047607422, + "fcm_dpo/q_t": 0.449276328086853, + "grad_norm": 40.509952545166016, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.9403315186500549, + "logits/rejected": -0.9331901669502258, + "logps/chosen": -859.654052734375, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1121.66064453125, + "loss": 1.2689, + "margin_dpo/margin_mean": 241.9705047607422, + "margin_dpo/margin_std": 691.712158203125, + "step": 482 + }, + { + "KL/chosen_KL_mean": -713.016845703125, + "KL/mean": -968.009033203125, + "KL/rejected_KL_mean": -1223.001220703125, + "KL/std": 581.701416015625, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0009073538822121918, + "fcm_dpo/delta": -0.07352857291698456, + "fcm_dpo/margin": 509.9843444824219, + "fcm_dpo/q_t": 0.3948502540588379, + "grad_norm": 29.643489837646484, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -1.0055943727493286, + "logits/rejected": -1.0237212181091309, + "logps/chosen": -776.3466796875, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1318.7880859375, + "loss": 1.0539, + "margin_dpo/margin_mean": 509.9843444824219, + "margin_dpo/margin_std": 650.2207641601562, + "step": 483 + }, + { + "KL/chosen_KL_mean": -605.034423828125, + "KL/mean": -836.6314086914062, + "KL/rejected_KL_mean": -1068.228515625, + "KL/std": 541.3726196289062, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0009063525940291584, + "fcm_dpo/delta": -0.023969601839780807, + "fcm_dpo/margin": 463.194091796875, + "fcm_dpo/q_t": 0.4030453562736511, + "grad_norm": 40.7518310546875, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -1.0230693817138672, + "logits/rejected": -1.0483049154281616, + "logps/chosen": -664.1724853515625, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1152.599853515625, + "loss": 1.0738, + "margin_dpo/margin_mean": 463.194091796875, + "margin_dpo/margin_std": 565.80126953125, + "step": 484 + }, + { + "KL/chosen_KL_mean": -620.41015625, + "KL/mean": -848.3262939453125, + "KL/rejected_KL_mean": -1076.242431640625, + "KL/std": 556.996337890625, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0008924601133912802, + "fcm_dpo/delta": -0.00808490440249443, + "fcm_dpo/margin": 455.8323059082031, + "fcm_dpo/q_t": 0.4083811044692993, + "grad_norm": 29.115455627441406, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.9556959867477417, + "logits/rejected": -0.9852679967880249, + "logps/chosen": -679.2597045898438, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1179.6065673828125, + "loss": 1.0984, + "margin_dpo/margin_mean": 455.83233642578125, + "margin_dpo/margin_std": 654.0283203125, + "step": 485 + }, + { + "KL/chosen_KL_mean": -717.9019775390625, + "KL/mean": -966.603759765625, + "KL/rejected_KL_mean": -1215.3056640625, + "KL/std": 641.1299438476562, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0008696732693351805, + "fcm_dpo/delta": -0.03746793419122696, + "fcm_dpo/margin": 497.4035949707031, + "fcm_dpo/q_t": 0.40244901180267334, + "grad_norm": 27.189159393310547, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -1.0456597805023193, + "logits/rejected": -1.0665757656097412, + "logps/chosen": -773.1616821289062, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1307.4449462890625, + "loss": 1.1044, + "margin_dpo/margin_mean": 497.403564453125, + "margin_dpo/margin_std": 781.9620361328125, + "step": 486 + }, + { + "KL/chosen_KL_mean": -751.0819091796875, + "KL/mean": -936.9486083984375, + "KL/rejected_KL_mean": -1122.815185546875, + "KL/std": 581.3290405273438, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0009029656066559255, + "fcm_dpo/delta": 0.06967591494321823, + "fcm_dpo/margin": 371.7332458496094, + "fcm_dpo/q_t": 0.4228770434856415, + "grad_norm": 28.094966888427734, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.9907674789428711, + "logits/rejected": -0.997085690498352, + "logps/chosen": -804.145263671875, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1215.23388671875, + "loss": 1.1485, + "margin_dpo/margin_mean": 371.7332458496094, + "margin_dpo/margin_std": 602.6510009765625, + "step": 487 + }, + { + "KL/chosen_KL_mean": -621.453369140625, + "KL/mean": -814.7838134765625, + "KL/rejected_KL_mean": -1008.1142578125, + "KL/std": 510.23480224609375, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0009319771779701114, + "fcm_dpo/delta": 0.04248833656311035, + "fcm_dpo/margin": 386.6607360839844, + "fcm_dpo/q_t": 0.417335569858551, + "grad_norm": 27.947431564331055, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.9225368499755859, + "logits/rejected": -0.9260050058364868, + "logps/chosen": -673.6815795898438, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1092.120849609375, + "loss": 1.1128, + "margin_dpo/margin_mean": 386.6607666015625, + "margin_dpo/margin_std": 511.94879150390625, + "step": 488 + }, + { + "KL/chosen_KL_mean": -566.1726684570312, + "KL/mean": -753.4268798828125, + "KL/rejected_KL_mean": -940.6810302734375, + "KL/std": 504.57073974609375, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.000954316696152091, + "fcm_dpo/delta": 0.04647769033908844, + "fcm_dpo/margin": 374.5083312988281, + "fcm_dpo/q_t": 0.4194645285606384, + "grad_norm": 26.182106018066406, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.9717486500740051, + "logits/rejected": -0.9709118604660034, + "logps/chosen": -622.162353515625, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -1020.0791625976562, + "loss": 1.1205, + "margin_dpo/margin_mean": 374.5083312988281, + "margin_dpo/margin_std": 533.9894409179688, + "step": 489 + }, + { + "KL/chosen_KL_mean": -676.06982421875, + "KL/mean": -876.908447265625, + "KL/rejected_KL_mean": -1077.7470703125, + "KL/std": 606.36279296875, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0009635513415560126, + "fcm_dpo/delta": 0.014292988926172256, + "fcm_dpo/margin": 401.6772155761719, + "fcm_dpo/q_t": 0.4143189489841461, + "grad_norm": 32.759490966796875, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -1.0197691917419434, + "logits/rejected": -1.0614256858825684, + "logps/chosen": -728.436279296875, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1188.1561279296875, + "loss": 1.1408, + "margin_dpo/margin_mean": 401.67724609375, + "margin_dpo/margin_std": 699.514892578125, + "step": 490 + }, + { + "KL/chosen_KL_mean": -622.8295288085938, + "KL/mean": -755.21484375, + "KL/rejected_KL_mean": -887.6002197265625, + "KL/std": 508.8851623535156, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.001024909084662795, + "fcm_dpo/delta": 0.13940833508968353, + "fcm_dpo/margin": 264.77069091796875, + "fcm_dpo/q_t": 0.4398823380470276, + "grad_norm": 30.720504760742188, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.9957999587059021, + "logits/rejected": -0.9903267621994019, + "logps/chosen": -682.94580078125, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -960.873046875, + "loss": 1.1984, + "margin_dpo/margin_mean": 264.77069091796875, + "margin_dpo/margin_std": 513.053466796875, + "step": 491 + }, + { + "KL/chosen_KL_mean": -640.137939453125, + "KL/mean": -771.9046630859375, + "KL/rejected_KL_mean": -903.6714477539062, + "KL/std": 502.74957275390625, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0010977047495543957, + "fcm_dpo/delta": 0.12156951427459717, + "fcm_dpo/margin": 263.5334777832031, + "fcm_dpo/q_t": 0.43709301948547363, + "grad_norm": 32.53826904296875, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -1.0538036823272705, + "logits/rejected": -1.0333051681518555, + "logps/chosen": -694.132080078125, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -976.3310546875, + "loss": 1.2155, + "margin_dpo/margin_mean": 263.53350830078125, + "margin_dpo/margin_std": 586.0464477539062, + "step": 492 + }, + { + "KL/chosen_KL_mean": -695.5394897460938, + "KL/mean": -849.3009033203125, + "KL/rejected_KL_mean": -1003.062255859375, + "KL/std": 573.692138671875, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.001128972857259214, + "fcm_dpo/delta": 0.057312898337841034, + "fcm_dpo/margin": 307.5227355957031, + "fcm_dpo/q_t": 0.4218721389770508, + "grad_norm": 42.675167083740234, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -1.0943255424499512, + "logits/rejected": -1.065833330154419, + "logps/chosen": -771.0367431640625, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1090.38525390625, + "loss": 1.1672, + "margin_dpo/margin_mean": 307.5227355957031, + "margin_dpo/margin_std": 560.1964721679688, + "step": 493 + }, + { + "KL/chosen_KL_mean": -552.0046997070312, + "KL/mean": -793.7294921875, + "KL/rejected_KL_mean": -1035.4542236328125, + "KL/std": 529.43212890625, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.0010871111880987883, + "fcm_dpo/delta": -0.14926618337631226, + "fcm_dpo/margin": 483.44952392578125, + "fcm_dpo/q_t": 0.377328485250473, + "grad_norm": 36.51863098144531, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.9460798501968384, + "logits/rejected": -0.984372615814209, + "logps/chosen": -593.364013671875, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1121.545654296875, + "loss": 1.0071, + "margin_dpo/margin_mean": 483.4494934082031, + "margin_dpo/margin_std": 539.3519287109375, + "step": 494 + }, + { + "KL/chosen_KL_mean": -615.56494140625, + "KL/mean": -817.6380004882812, + "KL/rejected_KL_mean": -1019.711181640625, + "KL/std": 520.802490234375, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0010533574968576431, + "fcm_dpo/delta": -0.029492903500795364, + "fcm_dpo/margin": 404.1462707519531, + "fcm_dpo/q_t": 0.4060450792312622, + "grad_norm": 35.57887649536133, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -1.042378306388855, + "logits/rejected": -1.0542176961898804, + "logps/chosen": -679.0999755859375, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1111.135498046875, + "loss": 1.1038, + "margin_dpo/margin_mean": 404.146240234375, + "margin_dpo/margin_std": 637.1739501953125, + "step": 495 + }, + { + "KL/chosen_KL_mean": -764.419189453125, + "KL/mean": -873.26220703125, + "KL/rejected_KL_mean": -982.105224609375, + "KL/std": 427.09600830078125, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0011281955521553755, + "fcm_dpo/delta": 0.1658795028924942, + "fcm_dpo/margin": 217.6861114501953, + "fcm_dpo/q_t": 0.4449210464954376, + "grad_norm": 64.04717254638672, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -1.0645802021026611, + "logits/rejected": -1.0390629768371582, + "logps/chosen": -837.0111083984375, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1066.4345703125, + "loss": 1.2281, + "margin_dpo/margin_mean": 217.68612670898438, + "margin_dpo/margin_std": 487.0423278808594, + "step": 496 + }, + { + "KL/chosen_KL_mean": -700.0052490234375, + "KL/mean": -810.6131591796875, + "KL/rejected_KL_mean": -921.2210693359375, + "KL/std": 509.990478515625, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0011564334854483604, + "fcm_dpo/delta": 0.0054458146914839745, + "fcm_dpo/margin": 221.21583557128906, + "fcm_dpo/q_t": 0.44188153743743896, + "grad_norm": 37.88853073120117, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.9942458868026733, + "logits/rejected": -0.9794371128082275, + "logps/chosen": -758.5992431640625, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -997.5094604492188, + "loss": 1.2228, + "margin_dpo/margin_mean": 221.21583557128906, + "margin_dpo/margin_std": 476.608642578125, + "step": 497 + }, + { + "KL/chosen_KL_mean": -671.4306030273438, + "KL/mean": -863.401123046875, + "KL/rejected_KL_mean": -1055.3717041015625, + "KL/std": 552.48095703125, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0011328622931614518, + "fcm_dpo/delta": -0.04037115350365639, + "fcm_dpo/margin": 383.9410400390625, + "fcm_dpo/q_t": 0.402023047208786, + "grad_norm": 33.13188171386719, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -1.0240213871002197, + "logits/rejected": -1.0133302211761475, + "logps/chosen": -742.63623046875, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1139.3297119140625, + "loss": 1.1131, + "margin_dpo/margin_mean": 383.9410400390625, + "margin_dpo/margin_std": 630.9346923828125, + "step": 498 + }, + { + "KL/chosen_KL_mean": -593.4116821289062, + "KL/mean": -823.5595703125, + "KL/rejected_KL_mean": -1053.70751953125, + "KL/std": 556.8739013671875, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0010691729839891195, + "fcm_dpo/delta": -0.10910817980766296, + "fcm_dpo/margin": 460.29583740234375, + "fcm_dpo/q_t": 0.3880062699317932, + "grad_norm": 33.5864372253418, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -1.0626204013824463, + "logits/rejected": -1.104695439338684, + "logps/chosen": -644.6668701171875, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1154.7861328125, + "loss": 1.051, + "margin_dpo/margin_mean": 460.29583740234375, + "margin_dpo/margin_std": 621.9307861328125, + "step": 499 + }, + { + "KL/chosen_KL_mean": -698.745361328125, + "KL/mean": -885.627197265625, + "KL/rejected_KL_mean": -1072.509033203125, + "KL/std": 484.72418212890625, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0010558380745351315, + "fcm_dpo/delta": 0.005998249165713787, + "fcm_dpo/margin": 373.76361083984375, + "fcm_dpo/q_t": 0.41088441014289856, + "grad_norm": 32.75845718383789, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -1.0371595621109009, + "logits/rejected": -1.0720511674880981, + "logps/chosen": -755.7728271484375, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1166.443115234375, + "loss": 1.1278, + "margin_dpo/margin_mean": 373.76361083984375, + "margin_dpo/margin_std": 608.8817138671875, + "step": 500 + }, + { + "KL/chosen_KL_mean": -610.792724609375, + "KL/mean": -812.1013793945312, + "KL/rejected_KL_mean": -1013.409912109375, + "KL/std": 512.5452270507812, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0010563272517174482, + "fcm_dpo/delta": -0.029246773570775986, + "fcm_dpo/margin": 402.6171875, + "fcm_dpo/q_t": 0.40524762868881226, + "grad_norm": 32.496646881103516, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -1.0043013095855713, + "logits/rejected": -0.9970067739486694, + "logps/chosen": -665.15234375, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1093.566650390625, + "loss": 1.1075, + "margin_dpo/margin_mean": 402.6171875, + "margin_dpo/margin_std": 631.4959716796875, + "step": 501 + }, + { + "KL/chosen_KL_mean": -550.539794921875, + "KL/mean": -748.93310546875, + "KL/rejected_KL_mean": -947.326416015625, + "KL/std": 470.5106201171875, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0010394630953669548, + "fcm_dpo/delta": -0.014231529086828232, + "fcm_dpo/margin": 396.78668212890625, + "fcm_dpo/q_t": 0.4065066874027252, + "grad_norm": 27.10480308532715, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.9303309321403503, + "logits/rejected": -0.9149893522262573, + "logps/chosen": -618.1403198242188, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1030.275146484375, + "loss": 1.0806, + "margin_dpo/margin_mean": 396.7866516113281, + "margin_dpo/margin_std": 514.7156982421875, + "step": 502 + }, + { + "KL/chosen_KL_mean": -547.2332153320312, + "KL/mean": -731.3643188476562, + "KL/rejected_KL_mean": -915.495361328125, + "KL/std": 458.7100830078125, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.0010366695933043957, + "fcm_dpo/delta": 0.020445335656404495, + "fcm_dpo/margin": 368.26214599609375, + "fcm_dpo/q_t": 0.41420885920524597, + "grad_norm": 24.963150024414062, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.9165388345718384, + "logits/rejected": -0.9066522717475891, + "logps/chosen": -602.3116455078125, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -998.0008544921875, + "loss": 1.1027, + "margin_dpo/margin_mean": 368.26214599609375, + "margin_dpo/margin_std": 504.50299072265625, + "step": 503 + }, + { + "KL/chosen_KL_mean": -574.23291015625, + "KL/mean": -796.791748046875, + "KL/rejected_KL_mean": -1019.3506469726562, + "KL/std": 518.6561279296875, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0010228645987808704, + "fcm_dpo/delta": -0.06390336900949478, + "fcm_dpo/margin": 445.1177978515625, + "fcm_dpo/q_t": 0.39565929770469666, + "grad_norm": 31.115022659301758, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.9814969897270203, + "logits/rejected": -1.0108094215393066, + "logps/chosen": -634.1986083984375, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1123.11279296875, + "loss": 1.0611, + "margin_dpo/margin_mean": 445.1177978515625, + "margin_dpo/margin_std": 558.47412109375, + "step": 504 + }, + { + "KL/chosen_KL_mean": -636.4125366210938, + "KL/mean": -748.5430297851562, + "KL/rejected_KL_mean": -860.6735229492188, + "KL/std": 474.9736328125, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0010875099105760455, + "fcm_dpo/delta": 0.16810765862464905, + "fcm_dpo/margin": 224.26092529296875, + "fcm_dpo/q_t": 0.44532668590545654, + "grad_norm": 38.010013580322266, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.9758407473564148, + "logits/rejected": -0.9497923851013184, + "logps/chosen": -712.5674438476562, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -949.2589111328125, + "loss": 1.2188, + "margin_dpo/margin_mean": 224.2609405517578, + "margin_dpo/margin_std": 457.3785400390625, + "step": 505 + }, + { + "KL/chosen_KL_mean": -506.07037353515625, + "KL/mean": -699.0255126953125, + "KL/rejected_KL_mean": -891.9805908203125, + "KL/std": 477.39874267578125, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.001087523763999343, + "fcm_dpo/delta": -0.024144649505615234, + "fcm_dpo/margin": 385.91021728515625, + "fcm_dpo/q_t": 0.4035499691963196, + "grad_norm": 32.145179748535156, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.93152916431427, + "logits/rejected": -0.9458725452423096, + "logps/chosen": -555.0308837890625, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -970.3956298828125, + "loss": 1.0883, + "margin_dpo/margin_mean": 385.9102478027344, + "margin_dpo/margin_std": 528.6674194335938, + "step": 506 + }, + { + "KL/chosen_KL_mean": -594.1497802734375, + "KL/mean": -764.1021728515625, + "KL/rejected_KL_mean": -934.0546875, + "KL/std": 555.0877075195312, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.001108657568693161, + "fcm_dpo/delta": 0.02613823302090168, + "fcm_dpo/margin": 339.90484619140625, + "fcm_dpo/q_t": 0.41895681619644165, + "grad_norm": 36.91230773925781, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -1.0078797340393066, + "logits/rejected": -1.0114067792892456, + "logps/chosen": -660.9512939453125, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1029.427490234375, + "loss": 1.1456, + "margin_dpo/margin_mean": 339.90484619140625, + "margin_dpo/margin_std": 610.3236694335938, + "step": 507 + }, + { + "KL/chosen_KL_mean": -636.5570678710938, + "KL/mean": -782.1719970703125, + "KL/rejected_KL_mean": -927.7869873046875, + "KL/std": 518.55615234375, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0011594663374125957, + "fcm_dpo/delta": 0.06823734939098358, + "fcm_dpo/margin": 291.22991943359375, + "fcm_dpo/q_t": 0.4267747104167938, + "grad_norm": 39.4781608581543, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.9649393558502197, + "logits/rejected": -0.9645048379898071, + "logps/chosen": -707.8605346679688, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1023.41455078125, + "loss": 1.196, + "margin_dpo/margin_mean": 291.22991943359375, + "margin_dpo/margin_std": 618.4113159179688, + "step": 508 + }, + { + "KL/chosen_KL_mean": -458.185546875, + "KL/mean": -616.9246826171875, + "KL/rejected_KL_mean": -775.663818359375, + "KL/std": 377.70526123046875, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0011886548018082976, + "fcm_dpo/delta": 0.02420021966099739, + "fcm_dpo/margin": 317.4782409667969, + "fcm_dpo/q_t": 0.41478192806243896, + "grad_norm": 27.9116153717041, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.870207667350769, + "logits/rejected": -0.8699750900268555, + "logps/chosen": -522.0045166015625, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -858.9202880859375, + "loss": 1.1163, + "margin_dpo/margin_mean": 317.478271484375, + "margin_dpo/margin_std": 464.2435302734375, + "step": 509 + }, + { + "KL/chosen_KL_mean": -571.6314697265625, + "KL/mean": -767.0484008789062, + "KL/rejected_KL_mean": -962.46533203125, + "KL/std": 443.20867919921875, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0011530937626957893, + "fcm_dpo/delta": -0.05847305804491043, + "fcm_dpo/margin": 390.8338623046875, + "fcm_dpo/q_t": 0.3938140571117401, + "grad_norm": 28.287324905395508, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.8105640411376953, + "logits/rejected": -0.8318732976913452, + "logps/chosen": -623.5098876953125, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1065.23046875, + "loss": 1.0424, + "margin_dpo/margin_mean": 390.8338623046875, + "margin_dpo/margin_std": 426.814453125, + "step": 510 + }, + { + "KL/chosen_KL_mean": -519.4014892578125, + "KL/mean": -680.0241088867188, + "KL/rejected_KL_mean": -840.6466064453125, + "KL/std": 459.0299072265625, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0011581326834857464, + "fcm_dpo/delta": 0.0312727615237236, + "fcm_dpo/margin": 321.24517822265625, + "fcm_dpo/q_t": 0.41676849126815796, + "grad_norm": 29.784637451171875, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.934990406036377, + "logits/rejected": -0.9425644874572754, + "logps/chosen": -579.6396484375, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -933.50341796875, + "loss": 1.1423, + "margin_dpo/margin_mean": 321.24517822265625, + "margin_dpo/margin_std": 557.01318359375, + "step": 511 + }, + { + "KL/chosen_KL_mean": -432.3377685546875, + "KL/mean": -643.03271484375, + "KL/rejected_KL_mean": -853.7275390625, + "KL/std": 419.27789306640625, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.001112624304369092, + "fcm_dpo/delta": -0.0808810144662857, + "fcm_dpo/margin": 421.38983154296875, + "fcm_dpo/q_t": 0.3888947665691376, + "grad_norm": 50.26760482788086, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.912682294845581, + "logits/rejected": -0.9345096349716187, + "logps/chosen": -487.2432861328125, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -935.6034545898438, + "loss": 1.024, + "margin_dpo/margin_mean": 421.3898010253906, + "margin_dpo/margin_std": 429.2408447265625, + "step": 512 + }, + { + "KL/chosen_KL_mean": -540.8709716796875, + "KL/mean": -694.052490234375, + "KL/rejected_KL_mean": -847.234130859375, + "KL/std": 415.0015869140625, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0010840664617717266, + "fcm_dpo/delta": -0.03174256533384323, + "fcm_dpo/margin": 306.3631286621094, + "fcm_dpo/q_t": 0.42287755012512207, + "grad_norm": 31.773454666137695, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.9648545384407043, + "logits/rejected": -0.9486473798751831, + "logps/chosen": -605.787353515625, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -923.2965698242188, + "loss": 1.1481, + "margin_dpo/margin_mean": 306.3631591796875, + "margin_dpo/margin_std": 476.7315979003906, + "step": 513 + }, + { + "KL/chosen_KL_mean": -577.4931640625, + "KL/mean": -729.2637939453125, + "KL/rejected_KL_mean": -881.034423828125, + "KL/std": 447.49114990234375, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.001123302150517702, + "fcm_dpo/delta": 0.06498480588197708, + "fcm_dpo/margin": 303.5412902832031, + "fcm_dpo/q_t": 0.4248642027378082, + "grad_norm": 35.63784408569336, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.9186625480651855, + "logits/rejected": -0.9098622798919678, + "logps/chosen": -651.7227172851562, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -959.97998046875, + "loss": 1.1489, + "margin_dpo/margin_mean": 303.54132080078125, + "margin_dpo/margin_std": 509.271484375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -501.829345703125, + "KL/mean": -627.9400634765625, + "KL/rejected_KL_mean": -754.0509033203125, + "KL/std": 399.02606201171875, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0011844097170978785, + "fcm_dpo/delta": 0.11122119426727295, + "fcm_dpo/margin": 252.2215118408203, + "fcm_dpo/q_t": 0.43131643533706665, + "grad_norm": 34.345375061035156, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.8840858936309814, + "logits/rejected": -0.8953117728233337, + "logps/chosen": -552.23095703125, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -841.1486206054688, + "loss": 1.1822, + "margin_dpo/margin_mean": 252.22149658203125, + "margin_dpo/margin_std": 472.7237243652344, + "step": 515 + }, + { + "KL/chosen_KL_mean": -535.0078735351562, + "KL/mean": -710.1063232421875, + "KL/rejected_KL_mean": -885.204833984375, + "KL/std": 437.9486083984375, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.0012052833335474133, + "fcm_dpo/delta": -0.025943826884031296, + "fcm_dpo/margin": 350.1969909667969, + "fcm_dpo/q_t": 0.4013558030128479, + "grad_norm": 40.559547424316406, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.967258870601654, + "logits/rejected": -0.996803343296051, + "logps/chosen": -599.657470703125, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -996.92724609375, + "loss": 1.0756, + "margin_dpo/margin_mean": 350.19696044921875, + "margin_dpo/margin_std": 438.65216064453125, + "step": 516 + }, + { + "KL/chosen_KL_mean": -556.8197631835938, + "KL/mean": -765.5140380859375, + "KL/rejected_KL_mean": -974.208251953125, + "KL/std": 482.8786315917969, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0011354258749634027, + "fcm_dpo/delta": -0.08690818399190903, + "fcm_dpo/margin": 417.3884582519531, + "fcm_dpo/q_t": 0.39224791526794434, + "grad_norm": 29.612672805786133, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.9158914685249329, + "logits/rejected": -0.9163025617599487, + "logps/chosen": -617.7333374023438, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1063.291259765625, + "loss": 1.0486, + "margin_dpo/margin_mean": 417.388427734375, + "margin_dpo/margin_std": 536.0867919921875, + "step": 517 + }, + { + "KL/chosen_KL_mean": -539.1517333984375, + "KL/mean": -710.9952392578125, + "KL/rejected_KL_mean": -882.8386840820312, + "KL/std": 400.31646728515625, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.0011233189143240452, + "fcm_dpo/delta": 0.015288008376955986, + "fcm_dpo/margin": 343.6869812011719, + "fcm_dpo/q_t": 0.4103432893753052, + "grad_norm": 32.68082809448242, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.9514377117156982, + "logits/rejected": -0.9688372015953064, + "logps/chosen": -596.6076049804688, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -968.1513671875, + "loss": 1.0992, + "margin_dpo/margin_mean": 343.68695068359375, + "margin_dpo/margin_std": 451.2240295410156, + "step": 518 + }, + { + "KL/chosen_KL_mean": -535.6795654296875, + "KL/mean": -666.5262451171875, + "KL/rejected_KL_mean": -797.3729248046875, + "KL/std": 386.3453063964844, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0011979549890384078, + "fcm_dpo/delta": 0.09352685511112213, + "fcm_dpo/margin": 261.693359375, + "fcm_dpo/q_t": 0.4288790225982666, + "grad_norm": 30.118946075439453, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.9110023975372314, + "logits/rejected": -0.8953202962875366, + "logps/chosen": -609.7428588867188, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -901.8170776367188, + "loss": 1.1774, + "margin_dpo/margin_mean": 261.6933898925781, + "margin_dpo/margin_std": 480.7813720703125, + "step": 519 + }, + { + "KL/chosen_KL_mean": -598.917724609375, + "KL/mean": -766.4161376953125, + "KL/rejected_KL_mean": -933.9144897460938, + "KL/std": 469.4677734375, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.001193053089082241, + "fcm_dpo/delta": -2.6889145374298096e-05, + "fcm_dpo/margin": 334.99676513671875, + "fcm_dpo/q_t": 0.41273581981658936, + "grad_norm": 32.80324935913086, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.8977741003036499, + "logits/rejected": -0.9020281434059143, + "logps/chosen": -669.217529296875, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -1033.8958740234375, + "loss": 1.1215, + "margin_dpo/margin_mean": 334.9967956542969, + "margin_dpo/margin_std": 535.9783325195312, + "step": 520 + }, + { + "KL/chosen_KL_mean": -533.1904296875, + "KL/mean": -721.6487426757812, + "KL/rejected_KL_mean": -910.1070556640625, + "KL/std": 468.2117614746094, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0011968240141868591, + "fcm_dpo/delta": -0.061240337789058685, + "fcm_dpo/margin": 376.9166564941406, + "fcm_dpo/q_t": 0.3963508903980255, + "grad_norm": 30.097801208496094, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.9672213196754456, + "logits/rejected": -0.9746675491333008, + "logps/chosen": -591.3333740234375, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -993.3876953125, + "loss": 1.0663, + "margin_dpo/margin_mean": 376.9166564941406, + "margin_dpo/margin_std": 476.39959716796875, + "step": 521 + }, + { + "KL/chosen_KL_mean": -619.34423828125, + "KL/mean": -788.1532592773438, + "KL/rejected_KL_mean": -956.9622802734375, + "KL/std": 499.09405517578125, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.001165403751656413, + "fcm_dpo/delta": 0.007348040584474802, + "fcm_dpo/margin": 337.61798095703125, + "fcm_dpo/q_t": 0.411703884601593, + "grad_norm": 27.243125915527344, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.9547539949417114, + "logits/rejected": -0.9584004878997803, + "logps/chosen": -665.8919067382812, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -1022.9761352539062, + "loss": 1.1233, + "margin_dpo/margin_mean": 337.61798095703125, + "margin_dpo/margin_std": 539.9661865234375, + "step": 522 + }, + { + "KL/chosen_KL_mean": -653.6292114257812, + "KL/mean": -835.531494140625, + "KL/rejected_KL_mean": -1017.4337768554688, + "KL/std": 570.88427734375, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0011498668463900685, + "fcm_dpo/delta": -0.021069111302495003, + "fcm_dpo/margin": 363.80462646484375, + "fcm_dpo/q_t": 0.4071430563926697, + "grad_norm": 39.10981750488281, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -1.0114126205444336, + "logits/rejected": -1.014816403388977, + "logps/chosen": -715.3988037109375, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1101.1951904296875, + "loss": 1.1192, + "margin_dpo/margin_mean": 363.80462646484375, + "margin_dpo/margin_std": 598.8515625, + "step": 523 + }, + { + "KL/chosen_KL_mean": -659.2566528320312, + "KL/mean": -849.02978515625, + "KL/rejected_KL_mean": -1038.802734375, + "KL/std": 547.9898071289062, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0011147886980324984, + "fcm_dpo/delta": -0.029202260076999664, + "fcm_dpo/margin": 379.546142578125, + "fcm_dpo/q_t": 0.40146321058273315, + "grad_norm": 29.979949951171875, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -1.0742989778518677, + "logits/rejected": -1.0649514198303223, + "logps/chosen": -737.3287353515625, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1120.104736328125, + "loss": 1.1006, + "margin_dpo/margin_mean": 379.546142578125, + "margin_dpo/margin_std": 556.9313354492188, + "step": 524 + }, + { + "KL/chosen_KL_mean": -635.953125, + "KL/mean": -870.348876953125, + "KL/rejected_KL_mean": -1104.74462890625, + "KL/std": 549.2996215820312, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0010578641667962074, + "fcm_dpo/delta": -0.11588429659605026, + "fcm_dpo/margin": 468.79168701171875, + "fcm_dpo/q_t": 0.3864228129386902, + "grad_norm": 32.33747482299805, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.9820036888122559, + "logits/rejected": -1.0219625234603882, + "logps/chosen": -686.7809448242188, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1204.797607421875, + "loss": 1.0336, + "margin_dpo/margin_mean": 468.79168701171875, + "margin_dpo/margin_std": 575.0751342773438, + "step": 525 + }, + { + "KL/chosen_KL_mean": -678.6172485351562, + "KL/mean": -910.9122924804688, + "KL/rejected_KL_mean": -1143.207275390625, + "KL/std": 540.3369140625, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0010234033688902855, + "fcm_dpo/delta": -0.08782743662595749, + "fcm_dpo/margin": 464.590087890625, + "fcm_dpo/q_t": 0.3891344666481018, + "grad_norm": 25.339265823364258, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -1.0304372310638428, + "logits/rejected": -1.029862642288208, + "logps/chosen": -741.7844848632812, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1229.5166015625, + "loss": 1.0589, + "margin_dpo/margin_mean": 464.5901184082031, + "margin_dpo/margin_std": 610.2327880859375, + "step": 526 + }, + { + "KL/chosen_KL_mean": -720.614013671875, + "KL/mean": -878.4454956054688, + "KL/rejected_KL_mean": -1036.2769775390625, + "KL/std": 568.42822265625, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0010312057565897703, + "fcm_dpo/delta": 0.08278854936361313, + "fcm_dpo/margin": 315.6629638671875, + "fcm_dpo/q_t": 0.42542749643325806, + "grad_norm": 36.522247314453125, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.2235097885131836, + "logits/rejected": -1.1922650337219238, + "logps/chosen": -791.74267578125, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1114.61962890625, + "loss": 1.1454, + "margin_dpo/margin_mean": 315.6629943847656, + "margin_dpo/margin_std": 475.9560546875, + "step": 527 + }, + { + "KL/chosen_KL_mean": -722.622314453125, + "KL/mean": -924.1132202148438, + "KL/rejected_KL_mean": -1125.604248046875, + "KL/std": 548.0302124023438, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0010381261818110943, + "fcm_dpo/delta": -0.021063534542918205, + "fcm_dpo/margin": 402.98187255859375, + "fcm_dpo/q_t": 0.4073488712310791, + "grad_norm": 36.32444381713867, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -1.1155059337615967, + "logits/rejected": -1.1032581329345703, + "logps/chosen": -790.7117309570312, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1219.51416015625, + "loss": 1.1117, + "margin_dpo/margin_mean": 402.98187255859375, + "margin_dpo/margin_std": 640.5030517578125, + "step": 528 + }, + { + "KL/chosen_KL_mean": -856.09423828125, + "KL/mean": -993.0452270507812, + "KL/rejected_KL_mean": -1129.996337890625, + "KL/std": 508.49993896484375, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0010470080887898803, + "fcm_dpo/delta": 0.016037527471780777, + "fcm_dpo/margin": 273.902099609375, + "fcm_dpo/q_t": 0.4351412355899811, + "grad_norm": 39.295719146728516, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.1617459058761597, + "logits/rejected": -1.1776933670043945, + "logps/chosen": -911.669189453125, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1219.205322265625, + "loss": 1.2244, + "margin_dpo/margin_mean": 273.902099609375, + "margin_dpo/margin_std": 627.4752807617188, + "step": 529 + }, + { + "KL/chosen_KL_mean": -744.7139892578125, + "KL/mean": -960.7333374023438, + "KL/rejected_KL_mean": -1176.752685546875, + "KL/std": 617.777099609375, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0010279042180627584, + "fcm_dpo/delta": -0.0507824644446373, + "fcm_dpo/margin": 432.0386962890625, + "fcm_dpo/q_t": 0.40311896800994873, + "grad_norm": 44.50559997558594, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -1.0886889696121216, + "logits/rejected": -1.0878679752349854, + "logps/chosen": -792.3153686523438, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1264.0372314453125, + "loss": 1.1298, + "margin_dpo/margin_mean": 432.03875732421875, + "margin_dpo/margin_std": 778.5751953125, + "step": 530 + }, + { + "KL/chosen_KL_mean": -857.3604736328125, + "KL/mean": -1031.2232666015625, + "KL/rejected_KL_mean": -1205.0859375, + "KL/std": 653.899658203125, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0010409655515104532, + "fcm_dpo/delta": 0.042144663631916046, + "fcm_dpo/margin": 347.72552490234375, + "fcm_dpo/q_t": 0.42089301347732544, + "grad_norm": 41.08179473876953, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.1941709518432617, + "logits/rejected": -1.1764004230499268, + "logps/chosen": -912.6065673828125, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1275.6920166015625, + "loss": 1.1507, + "margin_dpo/margin_mean": 347.72552490234375, + "margin_dpo/margin_std": 612.9656982421875, + "step": 531 + }, + { + "KL/chosen_KL_mean": -852.81201171875, + "KL/mean": -1043.74267578125, + "KL/rejected_KL_mean": -1234.673583984375, + "KL/std": 619.9586791992188, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0010193655034527183, + "fcm_dpo/delta": 0.007790520787239075, + "fcm_dpo/margin": 381.8615417480469, + "fcm_dpo/q_t": 0.4135794937610626, + "grad_norm": 69.79627990722656, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.1243914365768433, + "logits/rejected": -1.0988380908966064, + "logps/chosen": -923.0980224609375, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1321.264892578125, + "loss": 1.1736, + "margin_dpo/margin_mean": 381.86151123046875, + "margin_dpo/margin_std": 742.836181640625, + "step": 532 + }, + { + "KL/chosen_KL_mean": -743.2042236328125, + "KL/mean": -880.5979614257812, + "KL/rejected_KL_mean": -1017.9917602539062, + "KL/std": 519.1492919921875, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0010485948296263814, + "fcm_dpo/delta": 0.006735551171004772, + "fcm_dpo/margin": 274.78753662109375, + "fcm_dpo/q_t": 0.4356157183647156, + "grad_norm": 35.47322463989258, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -1.162676453590393, + "logits/rejected": -1.1423327922821045, + "logps/chosen": -808.059326171875, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1094.579833984375, + "loss": 1.2201, + "margin_dpo/margin_mean": 274.78753662109375, + "margin_dpo/margin_std": 629.4703979492188, + "step": 533 + }, + { + "KL/chosen_KL_mean": -791.9388427734375, + "KL/mean": -988.31494140625, + "KL/rejected_KL_mean": -1184.691162109375, + "KL/std": 582.1441650390625, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0010235793888568878, + "fcm_dpo/delta": -0.0064060017466545105, + "fcm_dpo/margin": 392.75225830078125, + "fcm_dpo/q_t": 0.40980264544487, + "grad_norm": 33.40141296386719, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.1154931783676147, + "logits/rejected": -1.1184356212615967, + "logps/chosen": -852.0582275390625, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1263.234619140625, + "loss": 1.1168, + "margin_dpo/margin_mean": 392.75225830078125, + "margin_dpo/margin_std": 601.301025390625, + "step": 534 + }, + { + "KL/chosen_KL_mean": -654.2332763671875, + "KL/mean": -877.8125, + "KL/rejected_KL_mean": -1101.3916015625, + "KL/std": 556.99658203125, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0010354998521506786, + "fcm_dpo/delta": -0.0749303549528122, + "fcm_dpo/margin": 447.158447265625, + "fcm_dpo/q_t": 0.39491868019104004, + "grad_norm": 26.491836547851562, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -1.0257909297943115, + "logits/rejected": -1.0344964265823364, + "logps/chosen": -708.5634765625, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1197.6993408203125, + "loss": 1.0666, + "margin_dpo/margin_mean": 447.158447265625, + "margin_dpo/margin_std": 583.52099609375, + "step": 535 + }, + { + "KL/chosen_KL_mean": -596.191650390625, + "KL/mean": -867.9818115234375, + "KL/rejected_KL_mean": -1139.7720947265625, + "KL/std": 624.9847412109375, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.000944139261264354, + "fcm_dpo/delta": -0.1337963044643402, + "fcm_dpo/margin": 543.5804443359375, + "fcm_dpo/q_t": 0.38659486174583435, + "grad_norm": 23.294593811035156, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -1.0098322629928589, + "logits/rejected": -1.049689769744873, + "logps/chosen": -643.2721557617188, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1228.869873046875, + "loss": 1.038, + "margin_dpo/margin_mean": 543.5804443359375, + "margin_dpo/margin_std": 743.013427734375, + "step": 536 + }, + { + "KL/chosen_KL_mean": -623.7998046875, + "KL/mean": -800.8599853515625, + "KL/rejected_KL_mean": -977.9202880859375, + "KL/std": 499.66790771484375, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0009655841859057546, + "fcm_dpo/delta": 0.06150486692786217, + "fcm_dpo/margin": 354.1204528808594, + "fcm_dpo/q_t": 0.4220527410507202, + "grad_norm": 37.724754333496094, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -1.0359020233154297, + "logits/rejected": -1.0185152292251587, + "logps/chosen": -681.5473022460938, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1048.358642578125, + "loss": 1.1589, + "margin_dpo/margin_mean": 354.1203918457031, + "margin_dpo/margin_std": 611.8524169921875, + "step": 537 + }, + { + "KL/chosen_KL_mean": -729.6046142578125, + "KL/mean": -934.816162109375, + "KL/rejected_KL_mean": -1140.02783203125, + "KL/std": 534.134765625, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0009555231081321836, + "fcm_dpo/delta": 0.0086207315325737, + "fcm_dpo/margin": 410.4232177734375, + "fcm_dpo/q_t": 0.41036975383758545, + "grad_norm": 26.796846389770508, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -1.0265477895736694, + "logits/rejected": -1.0126469135284424, + "logps/chosen": -796.0205078125, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1224.255859375, + "loss": 1.0985, + "margin_dpo/margin_mean": 410.4232177734375, + "margin_dpo/margin_std": 560.275390625, + "step": 538 + }, + { + "KL/chosen_KL_mean": -631.67626953125, + "KL/mean": -822.0889282226562, + "KL/rejected_KL_mean": -1012.50146484375, + "KL/std": 439.6746826171875, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0009817727841436863, + "fcm_dpo/delta": 0.028943350538611412, + "fcm_dpo/margin": 380.8251953125, + "fcm_dpo/q_t": 0.4142039120197296, + "grad_norm": 25.660640716552734, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -1.079002857208252, + "logits/rejected": -1.0755877494812012, + "logps/chosen": -690.1691284179688, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1104.35546875, + "loss": 1.1028, + "margin_dpo/margin_mean": 380.8251953125, + "margin_dpo/margin_std": 496.8641357421875, + "step": 539 + }, + { + "KL/chosen_KL_mean": -622.972900390625, + "KL/mean": -871.677978515625, + "KL/rejected_KL_mean": -1120.383056640625, + "KL/std": 554.2312622070312, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0009536813013255596, + "fcm_dpo/delta": -0.08658929169178009, + "fcm_dpo/margin": 497.4101257324219, + "fcm_dpo/q_t": 0.39369896054267883, + "grad_norm": 30.85984992980957, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.9671063423156738, + "logits/rejected": -0.9978018999099731, + "logps/chosen": -686.4554443359375, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1236.81298828125, + "loss": 1.0602, + "margin_dpo/margin_mean": 497.4101257324219, + "margin_dpo/margin_std": 663.313232421875, + "step": 540 + }, + { + "KL/chosen_KL_mean": -740.9237060546875, + "KL/mean": -867.052978515625, + "KL/rejected_KL_mean": -993.182373046875, + "KL/std": 494.847412109375, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0009405265445820987, + "fcm_dpo/delta": 0.018011199310421944, + "fcm_dpo/margin": 252.2586212158203, + "fcm_dpo/q_t": 0.4454342722892761, + "grad_norm": 42.582611083984375, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.1054792404174805, + "logits/rejected": -1.0991296768188477, + "logps/chosen": -819.2040405273438, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1096.4560546875, + "loss": 1.2404, + "margin_dpo/margin_mean": 252.2586212158203, + "margin_dpo/margin_std": 592.5184326171875, + "step": 541 + }, + { + "KL/chosen_KL_mean": -645.805419921875, + "KL/mean": -892.1180419921875, + "KL/rejected_KL_mean": -1138.4306640625, + "KL/std": 561.5740966796875, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.0008951720083132386, + "fcm_dpo/delta": -0.05349696800112724, + "fcm_dpo/margin": 492.62530517578125, + "fcm_dpo/q_t": 0.3986285924911499, + "grad_norm": 37.752227783203125, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -1.0938200950622559, + "logits/rejected": -1.1355630159378052, + "logps/chosen": -703.2904052734375, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1234.90576171875, + "loss": 1.0814, + "margin_dpo/margin_mean": 492.62530517578125, + "margin_dpo/margin_std": 670.388427734375, + "step": 542 + }, + { + "KL/chosen_KL_mean": -703.6412963867188, + "KL/mean": -877.56689453125, + "KL/rejected_KL_mean": -1051.4925537109375, + "KL/std": 643.8773803710938, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0009388466132804751, + "fcm_dpo/delta": 0.08163179457187653, + "fcm_dpo/margin": 347.85113525390625, + "fcm_dpo/q_t": 0.4351038932800293, + "grad_norm": 34.521270751953125, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -1.0069867372512817, + "logits/rejected": -1.0424442291259766, + "logps/chosen": -764.2587890625, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1157.091552734375, + "loss": 1.2022, + "margin_dpo/margin_mean": 347.85113525390625, + "margin_dpo/margin_std": 784.99462890625, + "step": 543 + }, + { + "KL/chosen_KL_mean": -714.5244140625, + "KL/mean": -921.8302001953125, + "KL/rejected_KL_mean": -1129.135986328125, + "KL/std": 534.1683959960938, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0009546782239340246, + "fcm_dpo/delta": 0.00464700348675251, + "fcm_dpo/margin": 414.611572265625, + "fcm_dpo/q_t": 0.41016703844070435, + "grad_norm": 29.81113052368164, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -1.025500774383545, + "logits/rejected": -1.052979826927185, + "logps/chosen": -774.1666870117188, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1230.090576171875, + "loss": 1.1098, + "margin_dpo/margin_mean": 414.6115417480469, + "margin_dpo/margin_std": 620.7162475585938, + "step": 544 + }, + { + "KL/chosen_KL_mean": -660.681884765625, + "KL/mean": -904.065673828125, + "KL/rejected_KL_mean": -1147.449462890625, + "KL/std": 550.4140625, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0009346397127956152, + "fcm_dpo/delta": -0.06354370713233948, + "fcm_dpo/margin": 486.7675476074219, + "fcm_dpo/q_t": 0.3960573077201843, + "grad_norm": 32.75130844116211, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.9349105358123779, + "logits/rejected": -0.9764214158058167, + "logps/chosen": -728.3305053710938, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1243.357421875, + "loss": 1.0614, + "margin_dpo/margin_mean": 486.7675476074219, + "margin_dpo/margin_std": 628.9329833984375, + "step": 545 + }, + { + "KL/chosen_KL_mean": -633.4560546875, + "KL/mean": -793.1614990234375, + "KL/rejected_KL_mean": -952.8670654296875, + "KL/std": 479.251953125, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.0009658208582550287, + "fcm_dpo/delta": 0.10081300139427185, + "fcm_dpo/margin": 319.41107177734375, + "fcm_dpo/q_t": 0.42853426933288574, + "grad_norm": 31.47951316833496, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.9926744699478149, + "logits/rejected": -0.983307957649231, + "logps/chosen": -684.2002563476562, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1034.7332763671875, + "loss": 1.1607, + "margin_dpo/margin_mean": 319.41107177734375, + "margin_dpo/margin_std": 507.9046630859375, + "step": 546 + }, + { + "KL/chosen_KL_mean": -636.900634765625, + "KL/mean": -851.5877685546875, + "KL/rejected_KL_mean": -1066.27490234375, + "KL/std": 535.674072265625, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0009757987572811544, + "fcm_dpo/delta": -0.0217515230178833, + "fcm_dpo/margin": 429.37432861328125, + "fcm_dpo/q_t": 0.4051220417022705, + "grad_norm": 45.7818489074707, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.9713867902755737, + "logits/rejected": -0.9919019937515259, + "logps/chosen": -710.5883178710938, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1157.036376953125, + "loss": 1.0891, + "margin_dpo/margin_mean": 429.37432861328125, + "margin_dpo/margin_std": 599.9996337890625, + "step": 547 + }, + { + "KL/chosen_KL_mean": -665.563232421875, + "KL/mean": -853.7537841796875, + "KL/rejected_KL_mean": -1041.9442138671875, + "KL/std": 563.1593017578125, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0009819652186706662, + "fcm_dpo/delta": 0.03422550857067108, + "fcm_dpo/margin": 376.381103515625, + "fcm_dpo/q_t": 0.4203721582889557, + "grad_norm": 27.766407012939453, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -1.0666818618774414, + "logits/rejected": -1.0734937191009521, + "logps/chosen": -730.8095703125, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1136.062255859375, + "loss": 1.1321, + "margin_dpo/margin_mean": 376.381103515625, + "margin_dpo/margin_std": 631.2123413085938, + "step": 548 + }, + { + "KL/chosen_KL_mean": -688.1954345703125, + "KL/mean": -835.6382446289062, + "KL/rejected_KL_mean": -983.0810546875, + "KL/std": 456.26123046875, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.000991131761111319, + "fcm_dpo/delta": 0.009627990424633026, + "fcm_dpo/margin": 294.88555908203125, + "fcm_dpo/q_t": 0.4332141876220703, + "grad_norm": 46.694393157958984, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -1.0694687366485596, + "logits/rejected": -1.0632838010787964, + "logps/chosen": -737.4077758789062, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1056.991455078125, + "loss": 1.1873, + "margin_dpo/margin_mean": 294.8855895996094, + "margin_dpo/margin_std": 548.8023071289062, + "step": 549 + }, + { + "KL/chosen_KL_mean": -686.2954711914062, + "KL/mean": -866.961181640625, + "KL/rejected_KL_mean": -1047.6268310546875, + "KL/std": 528.92626953125, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.001009407453238964, + "fcm_dpo/delta": 0.03965500369668007, + "fcm_dpo/margin": 361.33148193359375, + "fcm_dpo/q_t": 0.4173702597618103, + "grad_norm": 47.80479049682617, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.9808708429336548, + "logits/rejected": -1.0091683864593506, + "logps/chosen": -743.1024169921875, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1142.752685546875, + "loss": 1.1268, + "margin_dpo/margin_mean": 361.33148193359375, + "margin_dpo/margin_std": 557.3447265625, + "step": 550 + }, + { + "KL/chosen_KL_mean": -601.2125244140625, + "KL/mean": -891.6123046875, + "KL/rejected_KL_mean": -1182.01220703125, + "KL/std": 574.1927490234375, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0009347728919237852, + "fcm_dpo/delta": -0.1731090098619461, + "fcm_dpo/margin": 580.7996215820312, + "fcm_dpo/q_t": 0.37110352516174316, + "grad_norm": 49.77595138549805, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.9424033164978027, + "logits/rejected": -1.0059906244277954, + "logps/chosen": -660.31884765625, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1293.6849365234375, + "loss": 0.9818, + "margin_dpo/margin_mean": 580.7996215820312, + "margin_dpo/margin_std": 582.7924194335938, + "step": 551 + }, + { + "KL/chosen_KL_mean": -587.4052734375, + "KL/mean": -911.8726806640625, + "KL/rejected_KL_mean": -1236.340087890625, + "KL/std": 653.5662841796875, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0008254111744463444, + "fcm_dpo/delta": -0.17336127161979675, + "fcm_dpo/margin": 648.9348754882812, + "fcm_dpo/q_t": 0.3765319585800171, + "grad_norm": 42.92742919921875, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -1.0275975465774536, + "logits/rejected": -1.078658938407898, + "logps/chosen": -649.7598876953125, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1340.90234375, + "loss": 1.0074, + "margin_dpo/margin_mean": 648.9349365234375, + "margin_dpo/margin_std": 749.9534301757812, + "step": 552 + }, + { + "KL/chosen_KL_mean": -680.405029296875, + "KL/mean": -853.9312744140625, + "KL/rejected_KL_mean": -1027.4573974609375, + "KL/std": 540.0321044921875, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0008643935434520245, + "fcm_dpo/delta": 0.11026312410831451, + "fcm_dpo/margin": 347.0523681640625, + "fcm_dpo/q_t": 0.43361696600914, + "grad_norm": 24.716848373413086, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -1.0225926637649536, + "logits/rejected": -1.0210795402526855, + "logps/chosen": -748.6638793945312, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1125.5545654296875, + "loss": 1.1687, + "margin_dpo/margin_mean": 347.0523376464844, + "margin_dpo/margin_std": 588.7896728515625, + "step": 553 + }, + { + "KL/chosen_KL_mean": -696.2091064453125, + "KL/mean": -919.3488159179688, + "KL/rejected_KL_mean": -1142.488525390625, + "KL/std": 584.9552001953125, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0008977587567642331, + "fcm_dpo/delta": -0.0027929171919822693, + "fcm_dpo/margin": 446.279296875, + "fcm_dpo/q_t": 0.4115123152732849, + "grad_norm": 72.4686050415039, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.9726330041885376, + "logits/rejected": -0.9497363567352295, + "logps/chosen": -764.1568603515625, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1232.271240234375, + "loss": 1.1336, + "margin_dpo/margin_mean": 446.279296875, + "margin_dpo/margin_std": 754.9847412109375, + "step": 554 + }, + { + "KL/chosen_KL_mean": -715.6664428710938, + "KL/mean": -991.3941650390625, + "KL/rejected_KL_mean": -1267.121826171875, + "KL/std": 633.61474609375, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0008543593576177955, + "fcm_dpo/delta": -0.08262835443019867, + "fcm_dpo/margin": 551.4554443359375, + "fcm_dpo/q_t": 0.3960796892642975, + "grad_norm": 37.353092193603516, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.0480984449386597, + "logits/rejected": -1.1136038303375244, + "logps/chosen": -768.9969482421875, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1375.601318359375, + "loss": 1.0771, + "margin_dpo/margin_mean": 551.4554443359375, + "margin_dpo/margin_std": 814.7586669921875, + "step": 555 + }, + { + "KL/chosen_KL_mean": -713.4342041015625, + "KL/mean": -874.3201293945312, + "KL/rejected_KL_mean": -1035.2059326171875, + "KL/std": 491.67437744140625, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0008872256148606539, + "fcm_dpo/delta": 0.12526218593120575, + "fcm_dpo/margin": 321.7717590332031, + "fcm_dpo/q_t": 0.43519821763038635, + "grad_norm": 29.882976531982422, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -1.0518784523010254, + "logits/rejected": -1.0511382818222046, + "logps/chosen": -772.0787353515625, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1136.54638671875, + "loss": 1.1675, + "margin_dpo/margin_mean": 321.77178955078125, + "margin_dpo/margin_std": 497.0333251953125, + "step": 556 + }, + { + "KL/chosen_KL_mean": -766.984375, + "KL/mean": -1017.581298828125, + "KL/rejected_KL_mean": -1268.1783447265625, + "KL/std": 677.718505859375, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0008786104153841734, + "fcm_dpo/delta": -0.047422073781490326, + "fcm_dpo/margin": 501.1939697265625, + "fcm_dpo/q_t": 0.40180647373199463, + "grad_norm": 50.05023956298828, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.0853943824768066, + "logits/rejected": -1.1287027597427368, + "logps/chosen": -834.8250122070312, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1378.117919921875, + "loss": 1.098, + "margin_dpo/margin_mean": 501.1939697265625, + "margin_dpo/margin_std": 765.6815795898438, + "step": 557 + }, + { + "KL/chosen_KL_mean": -718.3128662109375, + "KL/mean": -1054.1265869140625, + "KL/rejected_KL_mean": -1389.940185546875, + "KL/std": 641.2049560546875, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0008126210886985064, + "fcm_dpo/delta": -0.17489278316497803, + "fcm_dpo/margin": 671.6273803710938, + "fcm_dpo/q_t": 0.36921441555023193, + "grad_norm": 30.443954467773438, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.181760549545288, + "logits/rejected": -1.2110426425933838, + "logps/chosen": -780.68115234375, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1492.101318359375, + "loss": 0.9915, + "margin_dpo/margin_mean": 671.6273803710938, + "margin_dpo/margin_std": 710.2800903320312, + "step": 558 + }, + { + "KL/chosen_KL_mean": -827.102294921875, + "KL/mean": -1108.54150390625, + "KL/rejected_KL_mean": -1389.9808349609375, + "KL/std": 682.766357421875, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0007799374870955944, + "fcm_dpo/delta": -0.044871099293231964, + "fcm_dpo/margin": 562.8785400390625, + "fcm_dpo/q_t": 0.4002946615219116, + "grad_norm": 23.679685592651367, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.2697944641113281, + "logits/rejected": -1.2819671630859375, + "logps/chosen": -887.8546142578125, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1483.423095703125, + "loss": 1.075, + "margin_dpo/margin_mean": 562.8785400390625, + "margin_dpo/margin_std": 761.1256103515625, + "step": 559 + }, + { + "KL/chosen_KL_mean": -756.6922607421875, + "KL/mean": -950.2930908203125, + "KL/rejected_KL_mean": -1143.89404296875, + "KL/std": 610.3768310546875, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0007979685324244201, + "fcm_dpo/delta": 0.10061949491500854, + "fcm_dpo/margin": 387.20172119140625, + "fcm_dpo/q_t": 0.43167370557785034, + "grad_norm": 29.75299644470215, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.9832611680030823, + "logits/rejected": -0.967193603515625, + "logps/chosen": -814.7960205078125, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1223.88525390625, + "loss": 1.1733, + "margin_dpo/margin_mean": 387.20172119140625, + "margin_dpo/margin_std": 686.7384033203125, + "step": 560 + }, + { + "KL/chosen_KL_mean": -872.140869140625, + "KL/mean": -1043.19482421875, + "KL/rejected_KL_mean": -1214.24853515625, + "KL/std": 600.6371459960938, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0008679333841428161, + "fcm_dpo/delta": 0.11161176860332489, + "fcm_dpo/margin": 342.10760498046875, + "fcm_dpo/q_t": 0.4331173896789551, + "grad_norm": 38.26835250854492, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.1811617612838745, + "logits/rejected": -1.2093584537506104, + "logps/chosen": -923.0538330078125, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1313.317138671875, + "loss": 1.2094, + "margin_dpo/margin_mean": 342.10760498046875, + "margin_dpo/margin_std": 755.661865234375, + "step": 561 + }, + { + "KL/chosen_KL_mean": -847.83447265625, + "KL/mean": -1083.31640625, + "KL/rejected_KL_mean": -1318.79833984375, + "KL/std": 669.3462524414062, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0008583711460232735, + "fcm_dpo/delta": -0.006965767592191696, + "fcm_dpo/margin": 470.96380615234375, + "fcm_dpo/q_t": 0.4099277853965759, + "grad_norm": 32.28083419799805, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.175123691558838, + "logits/rejected": -1.1898926496505737, + "logps/chosen": -907.2988891601562, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1415.3409423828125, + "loss": 1.1117, + "margin_dpo/margin_mean": 470.96380615234375, + "margin_dpo/margin_std": 726.025146484375, + "step": 562 + }, + { + "KL/chosen_KL_mean": -932.0515747070312, + "KL/mean": -1104.2039794921875, + "KL/rejected_KL_mean": -1276.3564453125, + "KL/std": 695.3826904296875, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0008607417112216353, + "fcm_dpo/delta": -0.042801667004823685, + "fcm_dpo/margin": 344.3048400878906, + "fcm_dpo/q_t": 0.42719000577926636, + "grad_norm": 46.74894332885742, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.1562349796295166, + "logits/rejected": -1.1637009382247925, + "logps/chosen": -992.6597900390625, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1370.924072265625, + "loss": 1.238, + "margin_dpo/margin_mean": 344.3048095703125, + "margin_dpo/margin_std": 815.4557495117188, + "step": 563 + }, + { + "KL/chosen_KL_mean": -834.01416015625, + "KL/mean": -1059.8499755859375, + "KL/rejected_KL_mean": -1285.685791015625, + "KL/std": 647.272216796875, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0008608861826360226, + "fcm_dpo/delta": 0.012117274105548859, + "fcm_dpo/margin": 451.67169189453125, + "fcm_dpo/q_t": 0.4128786027431488, + "grad_norm": 33.87059783935547, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.2041339874267578, + "logits/rejected": -1.198503017425537, + "logps/chosen": -908.2825927734375, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1378.924072265625, + "loss": 1.1083, + "margin_dpo/margin_mean": 451.67169189453125, + "margin_dpo/margin_std": 647.334228515625, + "step": 564 + }, + { + "KL/chosen_KL_mean": -907.7662963867188, + "KL/mean": -1137.48828125, + "KL/rejected_KL_mean": -1367.210205078125, + "KL/std": 751.8851318359375, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.000859650899656117, + "fcm_dpo/delta": 0.005679897032678127, + "fcm_dpo/margin": 459.44403076171875, + "fcm_dpo/q_t": 0.4161943793296814, + "grad_norm": 55.80940628051758, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.1720229387283325, + "logits/rejected": -1.176281213760376, + "logps/chosen": -976.7862548828125, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1452.9892578125, + "loss": 1.1664, + "margin_dpo/margin_mean": 459.44403076171875, + "margin_dpo/margin_std": 911.50390625, + "step": 565 + }, + { + "KL/chosen_KL_mean": -828.40185546875, + "KL/mean": -1095.2100830078125, + "KL/rejected_KL_mean": -1362.018310546875, + "KL/std": 734.9419555664062, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0008510933257639408, + "fcm_dpo/delta": -0.06400474905967712, + "fcm_dpo/margin": 533.6165161132812, + "fcm_dpo/q_t": 0.398329496383667, + "grad_norm": 35.09721755981445, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.1719138622283936, + "logits/rejected": -1.2015354633331299, + "logps/chosen": -894.9471435546875, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1465.8875732421875, + "loss": 1.1104, + "margin_dpo/margin_mean": 533.616455078125, + "margin_dpo/margin_std": 852.4638671875, + "step": 566 + }, + { + "KL/chosen_KL_mean": -774.268798828125, + "KL/mean": -978.67236328125, + "KL/rejected_KL_mean": -1183.075927734375, + "KL/std": 513.41650390625, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0008465960272587836, + "fcm_dpo/delta": 0.060260072350502014, + "fcm_dpo/margin": 408.80712890625, + "fcm_dpo/q_t": 0.41977089643478394, + "grad_norm": 28.013336181640625, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.2744722366333008, + "logits/rejected": -1.2877826690673828, + "logps/chosen": -827.1270751953125, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1268.44677734375, + "loss": 1.1219, + "margin_dpo/margin_mean": 408.80712890625, + "margin_dpo/margin_std": 553.5780639648438, + "step": 567 + }, + { + "KL/chosen_KL_mean": -762.24658203125, + "KL/mean": -1034.97900390625, + "KL/rejected_KL_mean": -1307.7113037109375, + "KL/std": 596.3646240234375, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0008296336745843291, + "fcm_dpo/delta": -0.061329081654548645, + "fcm_dpo/margin": 545.4647827148438, + "fcm_dpo/q_t": 0.39397096633911133, + "grad_norm": 31.42664337158203, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.1577560901641846, + "logits/rejected": -1.2049450874328613, + "logps/chosen": -807.43896484375, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1396.8037109375, + "loss": 1.0487, + "margin_dpo/margin_mean": 545.4647827148438, + "margin_dpo/margin_std": 639.5078125, + "step": 568 + }, + { + "KL/chosen_KL_mean": -854.2070922851562, + "KL/mean": -1079.9609375, + "KL/rejected_KL_mean": -1305.714599609375, + "KL/std": 756.03662109375, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0008490226464346051, + "fcm_dpo/delta": 0.015497211366891861, + "fcm_dpo/margin": 451.5075378417969, + "fcm_dpo/q_t": 0.412791907787323, + "grad_norm": 41.15495681762695, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.171088695526123, + "logits/rejected": -1.1812310218811035, + "logps/chosen": -917.9276733398438, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1384.81787109375, + "loss": 1.1458, + "margin_dpo/margin_mean": 451.50750732421875, + "margin_dpo/margin_std": 773.6370239257812, + "step": 569 + }, + { + "KL/chosen_KL_mean": -810.3162231445312, + "KL/mean": -1012.0115966796875, + "KL/rejected_KL_mean": -1213.706787109375, + "KL/std": 605.1414794921875, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0008537136018276215, + "fcm_dpo/delta": 0.06216207891702652, + "fcm_dpo/margin": 403.390625, + "fcm_dpo/q_t": 0.42319053411483765, + "grad_norm": 30.302461624145508, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.277374505996704, + "logits/rejected": -1.2716038227081299, + "logps/chosen": -871.9307861328125, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1295.8486328125, + "loss": 1.1457, + "margin_dpo/margin_mean": 403.390625, + "margin_dpo/margin_std": 654.912109375, + "step": 570 + }, + { + "KL/chosen_KL_mean": -781.6932373046875, + "KL/mean": -1018.225830078125, + "KL/rejected_KL_mean": -1254.75830078125, + "KL/std": 594.1085815429688, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0008577441330999136, + "fcm_dpo/delta": -0.00684717670083046, + "fcm_dpo/margin": 473.06512451171875, + "fcm_dpo/q_t": 0.4083845019340515, + "grad_norm": 36.52130889892578, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.1036077737808228, + "logits/rejected": -1.1239582300186157, + "logps/chosen": -834.747314453125, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1346.09521484375, + "loss": 1.1124, + "margin_dpo/margin_mean": 473.06512451171875, + "margin_dpo/margin_std": 732.7314453125, + "step": 571 + }, + { + "KL/chosen_KL_mean": -830.8375244140625, + "KL/mean": -1077.9237060546875, + "KL/rejected_KL_mean": -1325.010009765625, + "KL/std": 712.4929809570312, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0008471171604469419, + "fcm_dpo/delta": -0.021867651492357254, + "fcm_dpo/margin": 494.1724853515625, + "fcm_dpo/q_t": 0.4094105064868927, + "grad_norm": 32.31637954711914, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.1557081937789917, + "logits/rejected": -1.1922253370285034, + "logps/chosen": -879.2967529296875, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1408.567138671875, + "loss": 1.1262, + "margin_dpo/margin_mean": 494.17254638671875, + "margin_dpo/margin_std": 855.6088256835938, + "step": 572 + }, + { + "KL/chosen_KL_mean": -784.99560546875, + "KL/mean": -1009.668212890625, + "KL/rejected_KL_mean": -1234.3408203125, + "KL/std": 611.609130859375, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0008612012024968863, + "fcm_dpo/delta": 0.014380722306668758, + "fcm_dpo/margin": 449.34521484375, + "fcm_dpo/q_t": 0.4126865267753601, + "grad_norm": 31.42186164855957, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.1854324340820312, + "logits/rejected": -1.203126072883606, + "logps/chosen": -836.6182250976562, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1319.665771484375, + "loss": 1.1001, + "margin_dpo/margin_mean": 449.34521484375, + "margin_dpo/margin_std": 616.9964599609375, + "step": 573 + }, + { + "KL/chosen_KL_mean": -756.0161743164062, + "KL/mean": -998.3463134765625, + "KL/rejected_KL_mean": -1240.676513671875, + "KL/std": 577.7587890625, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0008573478553444147, + "fcm_dpo/delta": -0.01797374151647091, + "fcm_dpo/margin": 484.6602783203125, + "fcm_dpo/q_t": 0.40470442175865173, + "grad_norm": 29.371341705322266, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.2006587982177734, + "logits/rejected": -1.2352988719940186, + "logps/chosen": -807.0606689453125, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1333.48291015625, + "loss": 1.0917, + "margin_dpo/margin_mean": 484.6602783203125, + "margin_dpo/margin_std": 672.3667602539062, + "step": 574 + }, + { + "KL/chosen_KL_mean": -807.3775024414062, + "KL/mean": -1027.7115478515625, + "KL/rejected_KL_mean": -1248.0455322265625, + "KL/std": 591.8450927734375, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0008164903265424073, + "fcm_dpo/delta": -0.06450238078832626, + "fcm_dpo/margin": 440.66802978515625, + "fcm_dpo/q_t": 0.4161473214626312, + "grad_norm": 33.69388198852539, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -1.0412192344665527, + "logits/rejected": -1.0539319515228271, + "logps/chosen": -879.1676025390625, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1343.4317626953125, + "loss": 1.1502, + "margin_dpo/margin_mean": 440.6680603027344, + "margin_dpo/margin_std": 746.9760131835938, + "step": 575 + }, + { + "KL/chosen_KL_mean": -774.9896850585938, + "KL/mean": -939.5641479492188, + "KL/rejected_KL_mean": -1104.1385498046875, + "KL/std": 521.9180297851562, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.000832082936540246, + "fcm_dpo/delta": 0.03152836486697197, + "fcm_dpo/margin": 329.14886474609375, + "fcm_dpo/q_t": 0.4369269013404846, + "grad_norm": 28.40019416809082, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.1551799774169922, + "logits/rejected": -1.1910724639892578, + "logps/chosen": -829.252685546875, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1204.892822265625, + "loss": 1.1963, + "margin_dpo/margin_mean": 329.1488952636719, + "margin_dpo/margin_std": 620.4453125, + "step": 576 + }, + { + "KL/chosen_KL_mean": -677.2899169921875, + "KL/mean": -902.4747314453125, + "KL/rejected_KL_mean": -1127.659423828125, + "KL/std": 593.83935546875, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0008373443270102143, + "fcm_dpo/delta": 0.025751816108822823, + "fcm_dpo/margin": 450.3697509765625, + "fcm_dpo/q_t": 0.415219247341156, + "grad_norm": 29.91676902770996, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.1202982664108276, + "logits/rejected": -1.110447645187378, + "logps/chosen": -739.1995849609375, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1211.730224609375, + "loss": 1.1253, + "margin_dpo/margin_mean": 450.3697814941406, + "margin_dpo/margin_std": 672.1941528320312, + "step": 577 + }, + { + "KL/chosen_KL_mean": -655.10107421875, + "KL/mean": -893.4134521484375, + "KL/rejected_KL_mean": -1131.725830078125, + "KL/std": 578.6603393554688, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.0008327961550094187, + "fcm_dpo/delta": 0.0017990879714488983, + "fcm_dpo/margin": 476.6247253417969, + "fcm_dpo/q_t": 0.40841901302337646, + "grad_norm": 29.630420684814453, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.9847003221511841, + "logits/rejected": -1.0236585140228271, + "logps/chosen": -704.36474609375, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1215.162109375, + "loss": 1.0901, + "margin_dpo/margin_mean": 476.624755859375, + "margin_dpo/margin_std": 603.2166748046875, + "step": 578 + }, + { + "KL/chosen_KL_mean": -760.0106811523438, + "KL/mean": -906.692626953125, + "KL/rejected_KL_mean": -1053.37451171875, + "KL/std": 581.0877075195312, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0008737717871554196, + "fcm_dpo/delta": 0.053927645087242126, + "fcm_dpo/margin": 293.3638000488281, + "fcm_dpo/q_t": 0.44180336594581604, + "grad_norm": 52.18403244018555, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -1.0818220376968384, + "logits/rejected": -1.0881195068359375, + "logps/chosen": -818.896484375, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1148.162109375, + "loss": 1.2201, + "margin_dpo/margin_mean": 293.3638000488281, + "margin_dpo/margin_std": 638.9380493164062, + "step": 579 + }, + { + "KL/chosen_KL_mean": -534.255126953125, + "KL/mean": -805.421142578125, + "KL/rejected_KL_mean": -1076.5870361328125, + "KL/std": 637.5589599609375, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0008389790891669691, + "fcm_dpo/delta": -0.06549236923456192, + "fcm_dpo/margin": 542.3319702148438, + "fcm_dpo/q_t": 0.39787042140960693, + "grad_norm": 37.1919059753418, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8876287937164307, + "logits/rejected": -0.9244289398193359, + "logps/chosen": -582.9619140625, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1158.345458984375, + "loss": 1.064, + "margin_dpo/margin_mean": 542.33203125, + "margin_dpo/margin_std": 733.103759765625, + "step": 580 + }, + { + "KL/chosen_KL_mean": -752.3706665039062, + "KL/mean": -942.2613525390625, + "KL/rejected_KL_mean": -1132.152099609375, + "KL/std": 618.788330078125, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.0008700564503669739, + "fcm_dpo/delta": 0.07715247571468353, + "fcm_dpo/margin": 379.78131103515625, + "fcm_dpo/q_t": 0.42673540115356445, + "grad_norm": 27.62555694580078, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.1618289947509766, + "logits/rejected": -1.177741289138794, + "logps/chosen": -814.593017578125, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1223.8876953125, + "loss": 1.1606, + "margin_dpo/margin_mean": 379.7813720703125, + "margin_dpo/margin_std": 669.273681640625, + "step": 581 + }, + { + "KL/chosen_KL_mean": -657.2913818359375, + "KL/mean": -867.788330078125, + "KL/rejected_KL_mean": -1078.285400390625, + "KL/std": 520.4821166992188, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.00086187943816185, + "fcm_dpo/delta": -0.069728784263134, + "fcm_dpo/margin": 420.99395751953125, + "fcm_dpo/q_t": 0.4139789640903473, + "grad_norm": 30.715805053710938, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -1.0448081493377686, + "logits/rejected": -1.0742528438568115, + "logps/chosen": -715.52001953125, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1188.35498046875, + "loss": 1.1203, + "margin_dpo/margin_mean": 420.99395751953125, + "margin_dpo/margin_std": 562.4373779296875, + "step": 582 + }, + { + "KL/chosen_KL_mean": -737.9644775390625, + "KL/mean": -878.0505981445312, + "KL/rejected_KL_mean": -1018.13671875, + "KL/std": 579.1885986328125, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0008408981375396252, + "fcm_dpo/delta": -0.005845813080668449, + "fcm_dpo/margin": 280.172119140625, + "fcm_dpo/q_t": 0.44648897647857666, + "grad_norm": 53.685874938964844, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -1.0225450992584229, + "logits/rejected": -1.02388596534729, + "logps/chosen": -794.827392578125, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1106.54052734375, + "loss": 1.2506, + "margin_dpo/margin_mean": 280.172119140625, + "margin_dpo/margin_std": 689.326171875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -654.791259765625, + "KL/mean": -922.0772705078125, + "KL/rejected_KL_mean": -1189.36328125, + "KL/std": 593.0877075195312, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.000823289854452014, + "fcm_dpo/delta": -0.04625372588634491, + "fcm_dpo/margin": 534.572021484375, + "fcm_dpo/q_t": 0.39831721782684326, + "grad_norm": 33.174537658691406, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.0831246376037598, + "logits/rejected": -1.1085516214370728, + "logps/chosen": -711.69189453125, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1286.999267578125, + "loss": 1.0555, + "margin_dpo/margin_mean": 534.572021484375, + "margin_dpo/margin_std": 620.7254028320312, + "step": 584 + }, + { + "KL/chosen_KL_mean": -747.2343139648438, + "KL/mean": -989.4556884765625, + "KL/rejected_KL_mean": -1231.677001953125, + "KL/std": 664.7772216796875, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0008134059607982635, + "fcm_dpo/delta": 0.006310204043984413, + "fcm_dpo/margin": 484.4427795410156, + "fcm_dpo/q_t": 0.4107728898525238, + "grad_norm": 34.79314422607422, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.127608060836792, + "logits/rejected": -1.1627211570739746, + "logps/chosen": -808.208251953125, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1315.8465576171875, + "loss": 1.1243, + "margin_dpo/margin_mean": 484.4427490234375, + "margin_dpo/margin_std": 784.7545166015625, + "step": 585 + }, + { + "KL/chosen_KL_mean": -787.4011840820312, + "KL/mean": -944.1376953125, + "KL/rejected_KL_mean": -1100.874267578125, + "KL/std": 573.6338500976562, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0008704622159712017, + "fcm_dpo/delta": 0.13885736465454102, + "fcm_dpo/margin": 313.47314453125, + "fcm_dpo/q_t": 0.4389517903327942, + "grad_norm": 36.1743278503418, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.0928421020507812, + "logits/rejected": -1.1198155879974365, + "logps/chosen": -844.2567749023438, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1192.677001953125, + "loss": 1.2047, + "margin_dpo/margin_mean": 313.47314453125, + "margin_dpo/margin_std": 632.057373046875, + "step": 586 + }, + { + "KL/chosen_KL_mean": -543.4615478515625, + "KL/mean": -758.5667724609375, + "KL/rejected_KL_mean": -973.6720581054688, + "KL/std": 594.1635131835938, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0009038691059686244, + "fcm_dpo/delta": 0.012126509100198746, + "fcm_dpo/margin": 430.2104797363281, + "fcm_dpo/q_t": 0.4110638499259949, + "grad_norm": 39.22536087036133, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.1124536991119385, + "logits/rejected": -1.1474683284759521, + "logps/chosen": -588.1531982421875, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1056.2958984375, + "loss": 1.104, + "margin_dpo/margin_mean": 430.2104797363281, + "margin_dpo/margin_std": 601.779541015625, + "step": 587 + }, + { + "KL/chosen_KL_mean": -723.3095092773438, + "KL/mean": -904.1917724609375, + "KL/rejected_KL_mean": -1085.073974609375, + "KL/std": 527.1142578125, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0009192567085847259, + "fcm_dpo/delta": 0.07489893585443497, + "fcm_dpo/margin": 361.76446533203125, + "fcm_dpo/q_t": 0.42396652698516846, + "grad_norm": 33.81727981567383, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.066401720046997, + "logits/rejected": -1.0958185195922852, + "logps/chosen": -773.6044921875, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1192.44384765625, + "loss": 1.1471, + "margin_dpo/margin_mean": 361.76446533203125, + "margin_dpo/margin_std": 561.7902221679688, + "step": 588 + }, + { + "KL/chosen_KL_mean": -735.9842529296875, + "KL/mean": -986.2646484375, + "KL/rejected_KL_mean": -1236.545166015625, + "KL/std": 596.337646484375, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0009136195294559002, + "fcm_dpo/delta": -0.06640733033418655, + "fcm_dpo/margin": 500.5609130859375, + "fcm_dpo/q_t": 0.3946426510810852, + "grad_norm": 30.045812606811523, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.0677993297576904, + "logits/rejected": -1.0883740186691284, + "logps/chosen": -795.9141235351562, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1348.200439453125, + "loss": 1.0574, + "margin_dpo/margin_mean": 500.5609130859375, + "margin_dpo/margin_std": 635.906005859375, + "step": 589 + }, + { + "KL/chosen_KL_mean": -622.71875, + "KL/mean": -871.3973999023438, + "KL/rejected_KL_mean": -1120.0760498046875, + "KL/std": 599.819091796875, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0008802700904197991, + "fcm_dpo/delta": -0.04412468522787094, + "fcm_dpo/margin": 497.357421875, + "fcm_dpo/q_t": 0.4004269242286682, + "grad_norm": 27.163776397705078, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -1.043388843536377, + "logits/rejected": -1.0831242799758911, + "logps/chosen": -678.5285034179688, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1226.138916015625, + "loss": 1.0669, + "margin_dpo/margin_mean": 497.357421875, + "margin_dpo/margin_std": 640.720703125, + "step": 590 + }, + { + "KL/chosen_KL_mean": -682.65771484375, + "KL/mean": -910.385986328125, + "KL/rejected_KL_mean": -1138.1142578125, + "KL/std": 558.8634033203125, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0008796605980023742, + "fcm_dpo/delta": -0.0007357114227488637, + "fcm_dpo/margin": 455.45660400390625, + "fcm_dpo/q_t": 0.40696167945861816, + "grad_norm": 31.112394332885742, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.13508939743042, + "logits/rejected": -1.1401097774505615, + "logps/chosen": -738.8983154296875, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1221.900634765625, + "loss": 1.0968, + "margin_dpo/margin_mean": 455.45660400390625, + "margin_dpo/margin_std": 627.993896484375, + "step": 591 + }, + { + "KL/chosen_KL_mean": -714.1116943359375, + "KL/mean": -928.77783203125, + "KL/rejected_KL_mean": -1143.44384765625, + "KL/std": 580.65283203125, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0008866181015037, + "fcm_dpo/delta": 0.021857215091586113, + "fcm_dpo/margin": 429.3321533203125, + "fcm_dpo/q_t": 0.4145823121070862, + "grad_norm": 34.28858184814453, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.0985491275787354, + "logits/rejected": -1.1109273433685303, + "logps/chosen": -762.052001953125, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1219.1767578125, + "loss": 1.155, + "margin_dpo/margin_mean": 429.3321838378906, + "margin_dpo/margin_std": 769.8460693359375, + "step": 592 + }, + { + "KL/chosen_KL_mean": -690.8212890625, + "KL/mean": -852.568115234375, + "KL/rejected_KL_mean": -1014.31494140625, + "KL/std": 618.8264770507812, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0009014889365062118, + "fcm_dpo/delta": 0.019433340057730675, + "fcm_dpo/margin": 323.4936828613281, + "fcm_dpo/q_t": 0.43461930751800537, + "grad_norm": 40.54536819458008, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.9765288829803467, + "logits/rejected": -0.9734338521957397, + "logps/chosen": -739.5120239257812, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1075.222900390625, + "loss": 1.2059, + "margin_dpo/margin_mean": 323.49365234375, + "margin_dpo/margin_std": 681.3075561523438, + "step": 593 + }, + { + "KL/chosen_KL_mean": -659.693603515625, + "KL/mean": -859.9675903320312, + "KL/rejected_KL_mean": -1060.2415771484375, + "KL/std": 593.6884765625, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.000888380513060838, + "fcm_dpo/delta": -0.04918823763728142, + "fcm_dpo/margin": 400.5479736328125, + "fcm_dpo/q_t": 0.41752633452415466, + "grad_norm": 30.841676712036133, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.0619920492172241, + "logits/rejected": -1.0549980401992798, + "logps/chosen": -714.6284790039062, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1146.34130859375, + "loss": 1.1399, + "margin_dpo/margin_mean": 400.5479736328125, + "margin_dpo/margin_std": 619.5179443359375, + "step": 594 + }, + { + "KL/chosen_KL_mean": -619.4148559570312, + "KL/mean": -844.1845703125, + "KL/rejected_KL_mean": -1068.954345703125, + "KL/std": 554.01708984375, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0008729010005481541, + "fcm_dpo/delta": 0.008369861170649529, + "fcm_dpo/margin": 449.5394287109375, + "fcm_dpo/q_t": 0.4122171401977539, + "grad_norm": 50.13469314575195, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -1.05449378490448, + "logits/rejected": -1.084195852279663, + "logps/chosen": -668.8352661132812, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1149.58154296875, + "loss": 1.1011, + "margin_dpo/margin_mean": 449.53948974609375, + "margin_dpo/margin_std": 629.5330810546875, + "step": 595 + }, + { + "KL/chosen_KL_mean": -769.1572265625, + "KL/mean": -914.1146240234375, + "KL/rejected_KL_mean": -1059.072021484375, + "KL/std": 580.770751953125, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0009401568677276373, + "fcm_dpo/delta": 0.1378772258758545, + "fcm_dpo/margin": 289.9148864746094, + "fcm_dpo/q_t": 0.4427031874656677, + "grad_norm": 76.58032989501953, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -1.0561020374298096, + "logits/rejected": -1.0235692262649536, + "logps/chosen": -828.9488525390625, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1139.1632080078125, + "loss": 1.2328, + "margin_dpo/margin_mean": 289.91485595703125, + "margin_dpo/margin_std": 706.6992797851562, + "step": 596 + }, + { + "KL/chosen_KL_mean": -697.0907592773438, + "KL/mean": -974.411376953125, + "KL/rejected_KL_mean": -1251.73193359375, + "KL/std": 670.898681640625, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0008806366240605712, + "fcm_dpo/delta": -0.11443006992340088, + "fcm_dpo/margin": 554.6411743164062, + "fcm_dpo/q_t": 0.38930103182792664, + "grad_norm": 25.791946411132812, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -1.0218758583068848, + "logits/rejected": -1.0773537158966064, + "logps/chosen": -754.3515625, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1352.4256591796875, + "loss": 1.0594, + "margin_dpo/margin_mean": 554.6411743164062, + "margin_dpo/margin_std": 745.9192504882812, + "step": 597 + }, + { + "KL/chosen_KL_mean": -701.50537109375, + "KL/mean": -915.1785888671875, + "KL/rejected_KL_mean": -1128.851806640625, + "KL/std": 572.6109619140625, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0008801105432212353, + "fcm_dpo/delta": 0.0233195498585701, + "fcm_dpo/margin": 427.346435546875, + "fcm_dpo/q_t": 0.413480669260025, + "grad_norm": 42.76176452636719, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.0965592861175537, + "logits/rejected": -1.1154861450195312, + "logps/chosen": -754.0238647460938, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1218.295654296875, + "loss": 1.128, + "margin_dpo/margin_mean": 427.346435546875, + "margin_dpo/margin_std": 644.4198608398438, + "step": 598 + }, + { + "KL/chosen_KL_mean": -725.7476806640625, + "KL/mean": -907.746826171875, + "KL/rejected_KL_mean": -1089.74609375, + "KL/std": 552.6034545898438, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0008764672093093395, + "fcm_dpo/delta": -0.052862539887428284, + "fcm_dpo/margin": 363.9983825683594, + "fcm_dpo/q_t": 0.4220387637615204, + "grad_norm": 35.5477409362793, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -1.0401568412780762, + "logits/rejected": -1.0526893138885498, + "logps/chosen": -775.55029296875, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1172.724609375, + "loss": 1.148, + "margin_dpo/margin_mean": 363.9984130859375, + "margin_dpo/margin_std": 519.5889892578125, + "step": 599 + }, + { + "KL/chosen_KL_mean": -780.717529296875, + "KL/mean": -957.4244384765625, + "KL/rejected_KL_mean": -1134.13134765625, + "KL/std": 560.0313110351562, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0009075045818462968, + "fcm_dpo/delta": 0.08796217292547226, + "fcm_dpo/margin": 353.413818359375, + "fcm_dpo/q_t": 0.4293411076068878, + "grad_norm": 36.71418762207031, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.1779966354370117, + "logits/rejected": -1.1793086528778076, + "logps/chosen": -847.15234375, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1219.587890625, + "loss": 1.18, + "margin_dpo/margin_mean": 353.413818359375, + "margin_dpo/margin_std": 681.7435302734375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -756.00390625, + "KL/mean": -989.05517578125, + "KL/rejected_KL_mean": -1222.1064453125, + "KL/std": 601.011962890625, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.000907495035789907, + "fcm_dpo/delta": -0.026769058778882027, + "fcm_dpo/margin": 466.1025695800781, + "fcm_dpo/q_t": 0.4011099338531494, + "grad_norm": 33.50484085083008, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.0792961120605469, + "logits/rejected": -1.106907844543457, + "logps/chosen": -815.137451171875, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1316.79736328125, + "loss": 1.0759, + "margin_dpo/margin_mean": 466.1025695800781, + "margin_dpo/margin_std": 595.927490234375, + "step": 601 + }, + { + "KL/chosen_KL_mean": -541.6345825195312, + "KL/mean": -814.474609375, + "KL/rejected_KL_mean": -1087.314697265625, + "KL/std": 554.52001953125, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0008864919655025005, + "fcm_dpo/delta": -0.09840202331542969, + "fcm_dpo/margin": 545.6799926757812, + "fcm_dpo/q_t": 0.3876655697822571, + "grad_norm": 74.75506591796875, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -1.1055960655212402, + "logits/rejected": -1.1474847793579102, + "logps/chosen": -590.2281494140625, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1174.983154296875, + "loss": 1.0277, + "margin_dpo/margin_mean": 545.6799926757812, + "margin_dpo/margin_std": 600.5530395507812, + "step": 602 + }, + { + "KL/chosen_KL_mean": -684.0531616210938, + "KL/mean": -935.1064453125, + "KL/rejected_KL_mean": -1186.1597900390625, + "KL/std": 626.862060546875, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0008442359394393861, + "fcm_dpo/delta": -0.02752744033932686, + "fcm_dpo/margin": 502.10650634765625, + "fcm_dpo/q_t": 0.4052655100822449, + "grad_norm": 27.675668716430664, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -1.102555751800537, + "logits/rejected": -1.0997867584228516, + "logps/chosen": -754.4677734375, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1286.4853515625, + "loss": 1.0916, + "margin_dpo/margin_mean": 502.10650634765625, + "margin_dpo/margin_std": 729.0457763671875, + "step": 603 + }, + { + "KL/chosen_KL_mean": -663.9873046875, + "KL/mean": -930.0084228515625, + "KL/rejected_KL_mean": -1196.029541015625, + "KL/std": 591.8416748046875, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0008249485399574041, + "fcm_dpo/delta": -0.04492639750242233, + "fcm_dpo/margin": 532.0422973632812, + "fcm_dpo/q_t": 0.40083804726600647, + "grad_norm": 41.498619079589844, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -1.104397177696228, + "logits/rejected": -1.1467612981796265, + "logps/chosen": -710.4453735351562, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1287.884033203125, + "loss": 1.0818, + "margin_dpo/margin_mean": 532.0422973632812, + "margin_dpo/margin_std": 747.6170043945312, + "step": 604 + }, + { + "KL/chosen_KL_mean": -705.8027954101562, + "KL/mean": -904.9942626953125, + "KL/rejected_KL_mean": -1104.185546875, + "KL/std": 564.7620849609375, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0008404471445828676, + "fcm_dpo/delta": 0.07256826758384705, + "fcm_dpo/margin": 398.3828125, + "fcm_dpo/q_t": 0.4260856807231903, + "grad_norm": 35.63572692871094, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -1.0245356559753418, + "logits/rejected": -1.0403711795806885, + "logps/chosen": -772.0521240234375, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1206.4906005859375, + "loss": 1.1541, + "margin_dpo/margin_mean": 398.3828125, + "margin_dpo/margin_std": 679.633056640625, + "step": 605 + }, + { + "KL/chosen_KL_mean": -695.2269287109375, + "KL/mean": -914.662109375, + "KL/rejected_KL_mean": -1134.097412109375, + "KL/std": 560.8505859375, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0008736621821299195, + "fcm_dpo/delta": 0.017634302377700806, + "fcm_dpo/margin": 438.87042236328125, + "fcm_dpo/q_t": 0.41308021545410156, + "grad_norm": 26.252599716186523, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -1.0582551956176758, + "logits/rejected": -1.077307939529419, + "logps/chosen": -750.0460205078125, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1232.46875, + "loss": 1.1057, + "margin_dpo/margin_mean": 438.87042236328125, + "margin_dpo/margin_std": 612.81982421875, + "step": 606 + }, + { + "KL/chosen_KL_mean": -739.9675903320312, + "KL/mean": -920.4398193359375, + "KL/rejected_KL_mean": -1100.9119873046875, + "KL/std": 599.8391723632812, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.000907676643691957, + "fcm_dpo/delta": 0.07830701023340225, + "fcm_dpo/margin": 360.9443664550781, + "fcm_dpo/q_t": 0.42693376541137695, + "grad_norm": 29.625747680664062, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -1.038463830947876, + "logits/rejected": -1.028611660003662, + "logps/chosen": -798.0516357421875, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1180.6890869140625, + "loss": 1.1632, + "margin_dpo/margin_mean": 360.94439697265625, + "margin_dpo/margin_std": 634.8746337890625, + "step": 607 + }, + { + "KL/chosen_KL_mean": -654.652587890625, + "KL/mean": -871.2743530273438, + "KL/rejected_KL_mean": -1087.896240234375, + "KL/std": 537.78466796875, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0009169199038296938, + "fcm_dpo/delta": 0.0025679636746644974, + "fcm_dpo/margin": 433.2436218261719, + "fcm_dpo/q_t": 0.40801483392715454, + "grad_norm": 35.430641174316406, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.0829113721847534, + "logits/rejected": -1.1019339561462402, + "logps/chosen": -712.1033935546875, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1182.6695556640625, + "loss": 1.0873, + "margin_dpo/margin_mean": 433.24359130859375, + "margin_dpo/margin_std": 546.6224365234375, + "step": 608 + }, + { + "KL/chosen_KL_mean": -683.7957763671875, + "KL/mean": -963.6317138671875, + "KL/rejected_KL_mean": -1243.4677734375, + "KL/std": 729.7833862304688, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.000869814888574183, + "fcm_dpo/delta": -0.10166175663471222, + "fcm_dpo/margin": 559.6719970703125, + "fcm_dpo/q_t": 0.3958240747451782, + "grad_norm": 28.936141967773438, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.1562999486923218, + "logits/rejected": -1.1883039474487305, + "logps/chosen": -742.6011352539062, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1332.28369140625, + "loss": 1.074, + "margin_dpo/margin_mean": 559.6719970703125, + "margin_dpo/margin_std": 881.8450317382812, + "step": 609 + }, + { + "KL/chosen_KL_mean": -665.3502197265625, + "KL/mean": -841.2243041992188, + "KL/rejected_KL_mean": -1017.0985107421875, + "KL/std": 539.9099731445312, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.000896269571967423, + "fcm_dpo/delta": 0.09289417415857315, + "fcm_dpo/margin": 351.74822998046875, + "fcm_dpo/q_t": 0.4279220402240753, + "grad_norm": 38.34468078613281, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -1.0509302616119385, + "logits/rejected": -1.0315933227539062, + "logps/chosen": -731.0452880859375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1100.50390625, + "loss": 1.1704, + "margin_dpo/margin_mean": 351.74822998046875, + "margin_dpo/margin_std": 631.10302734375, + "step": 610 + }, + { + "KL/chosen_KL_mean": -695.197509765625, + "KL/mean": -987.370361328125, + "KL/rejected_KL_mean": -1279.543212890625, + "KL/std": 713.4209594726562, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0008558118133805692, + "fcm_dpo/delta": -0.1183016374707222, + "fcm_dpo/margin": 584.3458251953125, + "fcm_dpo/q_t": 0.38911253213882446, + "grad_norm": 28.481924057006836, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.1229252815246582, + "logits/rejected": -1.1791808605194092, + "logps/chosen": -747.7969360351562, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1365.874267578125, + "loss": 1.0533, + "margin_dpo/margin_mean": 584.3457641601562, + "margin_dpo/margin_std": 811.1268310546875, + "step": 611 + }, + { + "KL/chosen_KL_mean": -754.8856201171875, + "KL/mean": -965.2699584960938, + "KL/rejected_KL_mean": -1175.654296875, + "KL/std": 569.8728637695312, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0008665896020829678, + "fcm_dpo/delta": 0.03841254860162735, + "fcm_dpo/margin": 420.76873779296875, + "fcm_dpo/q_t": 0.41685962677001953, + "grad_norm": 30.982759475708008, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -1.0996251106262207, + "logits/rejected": -1.104633092880249, + "logps/chosen": -814.2093505859375, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1263.9666748046875, + "loss": 1.1233, + "margin_dpo/margin_mean": 420.76873779296875, + "margin_dpo/margin_std": 623.3631591796875, + "step": 612 + }, + { + "KL/chosen_KL_mean": -664.0328369140625, + "KL/mean": -946.47998046875, + "KL/rejected_KL_mean": -1228.9271240234375, + "KL/std": 637.9658813476562, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.000826348434202373, + "fcm_dpo/delta": -0.0787663459777832, + "fcm_dpo/margin": 564.894287109375, + "fcm_dpo/q_t": 0.3913313150405884, + "grad_norm": 29.40831756591797, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -1.0969492197036743, + "logits/rejected": -1.1602132320404053, + "logps/chosen": -723.7628173828125, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1334.03466796875, + "loss": 1.0404, + "margin_dpo/margin_mean": 564.894287109375, + "margin_dpo/margin_std": 660.2083740234375, + "step": 613 + }, + { + "KL/chosen_KL_mean": -747.9976196289062, + "KL/mean": -1017.033935546875, + "KL/rejected_KL_mean": -1286.0703125, + "KL/std": 654.758544921875, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0008089347975328565, + "fcm_dpo/delta": -0.040523845702409744, + "fcm_dpo/margin": 538.07275390625, + "fcm_dpo/q_t": 0.40103471279144287, + "grad_norm": 40.31919860839844, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -1.049826741218567, + "logits/rejected": -1.1347273588180542, + "logps/chosen": -800.9365844726562, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1390.7496337890625, + "loss": 1.0902, + "margin_dpo/margin_mean": 538.0726928710938, + "margin_dpo/margin_std": 791.9971923828125, + "step": 614 + }, + { + "KL/chosen_KL_mean": -687.3709106445312, + "KL/mean": -908.230712890625, + "KL/rejected_KL_mean": -1129.090576171875, + "KL/std": 647.798828125, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0008249276434071362, + "fcm_dpo/delta": 0.03885895386338234, + "fcm_dpo/margin": 441.7195739746094, + "fcm_dpo/q_t": 0.41741758584976196, + "grad_norm": 26.804555892944336, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -1.0640817880630493, + "logits/rejected": -1.093153715133667, + "logps/chosen": -753.188232421875, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1224.26806640625, + "loss": 1.1335, + "margin_dpo/margin_mean": 441.7195739746094, + "margin_dpo/margin_std": 709.6761474609375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -832.925048828125, + "KL/mean": -1012.5926513671875, + "KL/rejected_KL_mean": -1192.26025390625, + "KL/std": 561.084228515625, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0008603649912402034, + "fcm_dpo/delta": 0.09993337839841843, + "fcm_dpo/margin": 359.3353576660156, + "fcm_dpo/q_t": 0.43038344383239746, + "grad_norm": 34.5568733215332, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.1497546434402466, + "logits/rejected": -1.1363475322723389, + "logps/chosen": -898.057861328125, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1266.9609375, + "loss": 1.1683, + "margin_dpo/margin_mean": 359.33538818359375, + "margin_dpo/margin_std": 615.418701171875, + "step": 616 + }, + { + "KL/chosen_KL_mean": -794.43310546875, + "KL/mean": -935.2199096679688, + "KL/rejected_KL_mean": -1076.0067138671875, + "KL/std": 541.37060546875, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0009373525972478092, + "fcm_dpo/delta": 0.1464797705411911, + "fcm_dpo/margin": 281.5736083984375, + "fcm_dpo/q_t": 0.4427611231803894, + "grad_norm": 57.17730712890625, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -1.0719687938690186, + "logits/rejected": -1.044560194015503, + "logps/chosen": -857.4385986328125, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1140.2408447265625, + "loss": 1.2245, + "margin_dpo/margin_mean": 281.5736083984375, + "margin_dpo/margin_std": 640.4603271484375, + "step": 617 + }, + { + "KL/chosen_KL_mean": -817.8272094726562, + "KL/mean": -1103.8603515625, + "KL/rejected_KL_mean": -1389.893310546875, + "KL/std": 714.6287231445312, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0009096855064854026, + "fcm_dpo/delta": -0.14220011234283447, + "fcm_dpo/margin": 572.066162109375, + "fcm_dpo/q_t": 0.3869177997112274, + "grad_norm": 35.31268310546875, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -1.1040403842926025, + "logits/rejected": -1.1524155139923096, + "logps/chosen": -884.9285888671875, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1482.0467529296875, + "loss": 1.0934, + "margin_dpo/margin_mean": 572.066162109375, + "margin_dpo/margin_std": 939.2500610351562, + "step": 618 + }, + { + "KL/chosen_KL_mean": -786.8121948242188, + "KL/mean": -1008.24951171875, + "KL/rejected_KL_mean": -1229.6868896484375, + "KL/std": 659.664794921875, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0008808445418253541, + "fcm_dpo/delta": 0.01115130539983511, + "fcm_dpo/margin": 442.87469482421875, + "fcm_dpo/q_t": 0.419704794883728, + "grad_norm": 48.612037658691406, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.0768239498138428, + "logits/rejected": -1.0969083309173584, + "logps/chosen": -842.7904052734375, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1322.872314453125, + "loss": 1.1651, + "margin_dpo/margin_mean": 442.87469482421875, + "margin_dpo/margin_std": 881.3223266601562, + "step": 619 + }, + { + "KL/chosen_KL_mean": -725.1968994140625, + "KL/mean": -927.8970947265625, + "KL/rejected_KL_mean": -1130.597412109375, + "KL/std": 568.570556640625, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0008872602484188974, + "fcm_dpo/delta": 0.0446610301733017, + "fcm_dpo/margin": 405.4005432128906, + "fcm_dpo/q_t": 0.41971608996391296, + "grad_norm": 32.381141662597656, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.1273236274719238, + "logits/rejected": -1.1392064094543457, + "logps/chosen": -784.994384765625, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1209.0081787109375, + "loss": 1.1361, + "margin_dpo/margin_mean": 405.4005432128906, + "margin_dpo/margin_std": 644.727294921875, + "step": 620 + }, + { + "KL/chosen_KL_mean": -716.549072265625, + "KL/mean": -1022.1708374023438, + "KL/rejected_KL_mean": -1327.7926025390625, + "KL/std": 696.8282470703125, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0008660799358040094, + "fcm_dpo/delta": -0.1540810763835907, + "fcm_dpo/margin": 611.2435302734375, + "fcm_dpo/q_t": 0.3776348829269409, + "grad_norm": 38.35250473022461, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.175751805305481, + "logits/rejected": -1.234482765197754, + "logps/chosen": -770.4827880859375, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1416.162109375, + "loss": 1.0323, + "margin_dpo/margin_mean": 611.2435302734375, + "margin_dpo/margin_std": 765.3636474609375, + "step": 621 + }, + { + "KL/chosen_KL_mean": -716.1577758789062, + "KL/mean": -930.5115966796875, + "KL/rejected_KL_mean": -1144.8653564453125, + "KL/std": 539.96630859375, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0008321298519149423, + "fcm_dpo/delta": 0.04797040671110153, + "fcm_dpo/margin": 428.70758056640625, + "fcm_dpo/q_t": 0.4200412631034851, + "grad_norm": 25.105064392089844, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -1.040635108947754, + "logits/rejected": -1.031171441078186, + "logps/chosen": -776.443603515625, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1230.3841552734375, + "loss": 1.1338, + "margin_dpo/margin_mean": 428.70758056640625, + "margin_dpo/margin_std": 672.3571166992188, + "step": 622 + }, + { + "KL/chosen_KL_mean": -789.953125, + "KL/mean": -946.2022705078125, + "KL/rejected_KL_mean": -1102.451416015625, + "KL/std": 564.91943359375, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0008990369969978929, + "fcm_dpo/delta": 0.1300329566001892, + "fcm_dpo/margin": 312.498291015625, + "fcm_dpo/q_t": 0.44014668464660645, + "grad_norm": 41.02130889892578, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.1784584522247314, + "logits/rejected": -1.179451823234558, + "logps/chosen": -854.110107421875, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1187.534423828125, + "loss": 1.2133, + "margin_dpo/margin_mean": 312.498291015625, + "margin_dpo/margin_std": 690.1802368164062, + "step": 623 + }, + { + "KL/chosen_KL_mean": -771.953369140625, + "KL/mean": -1033.3436279296875, + "KL/rejected_KL_mean": -1294.73388671875, + "KL/std": 597.0548095703125, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0008907003793865442, + "fcm_dpo/delta": -0.0761176198720932, + "fcm_dpo/margin": 522.7806396484375, + "fcm_dpo/q_t": 0.39050906896591187, + "grad_norm": 31.780357360839844, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.1616058349609375, + "logits/rejected": -1.1715960502624512, + "logps/chosen": -843.8720092773438, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1391.865966796875, + "loss": 1.0541, + "margin_dpo/margin_mean": 522.7805786132812, + "margin_dpo/margin_std": 655.8362426757812, + "step": 624 + }, + { + "KL/chosen_KL_mean": -731.2457885742188, + "KL/mean": -1012.51904296875, + "KL/rejected_KL_mean": -1293.7923583984375, + "KL/std": 638.5082397460938, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0008487566374242306, + "fcm_dpo/delta": -0.09013813734054565, + "fcm_dpo/margin": 562.5465087890625, + "fcm_dpo/q_t": 0.38838595151901245, + "grad_norm": 55.579830169677734, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -1.0656121969223022, + "logits/rejected": -1.0959112644195557, + "logps/chosen": -789.587890625, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1379.8826904296875, + "loss": 1.0226, + "margin_dpo/margin_mean": 562.5465087890625, + "margin_dpo/margin_std": 593.01611328125, + "step": 625 + }, + { + "KL/chosen_KL_mean": -889.0275268554688, + "KL/mean": -1046.1427001953125, + "KL/rejected_KL_mean": -1203.2578125, + "KL/std": 692.0028686523438, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0008827048004604876, + "fcm_dpo/delta": 0.1335112750530243, + "fcm_dpo/margin": 314.23028564453125, + "fcm_dpo/q_t": 0.43665677309036255, + "grad_norm": 31.135826110839844, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -1.0140643119812012, + "logits/rejected": -1.0087287425994873, + "logps/chosen": -964.14013671875, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1302.446533203125, + "loss": 1.2199, + "margin_dpo/margin_mean": 314.23028564453125, + "margin_dpo/margin_std": 686.618408203125, + "step": 626 + }, + { + "KL/chosen_KL_mean": -608.5745849609375, + "KL/mean": -923.7266235351562, + "KL/rejected_KL_mean": -1238.878662109375, + "KL/std": 745.49853515625, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0008527673780918121, + "fcm_dpo/delta": -0.1632433533668518, + "fcm_dpo/margin": 630.3041381835938, + "fcm_dpo/q_t": 0.38402342796325684, + "grad_norm": 28.532012939453125, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.9967671036720276, + "logits/rejected": -1.0804929733276367, + "logps/chosen": -656.3177490234375, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1345.6331787109375, + "loss": 1.036, + "margin_dpo/margin_mean": 630.3041381835938, + "margin_dpo/margin_std": 856.10595703125, + "step": 627 + }, + { + "KL/chosen_KL_mean": -792.1390380859375, + "KL/mean": -1027.471435546875, + "KL/rejected_KL_mean": -1262.803955078125, + "KL/std": 620.2369384765625, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0008179534343071282, + "fcm_dpo/delta": 0.016899362206459045, + "fcm_dpo/margin": 470.6648864746094, + "fcm_dpo/q_t": 0.4131387174129486, + "grad_norm": 30.864870071411133, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.1643486022949219, + "logits/rejected": -1.1849493980407715, + "logps/chosen": -852.3219604492188, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1364.3585205078125, + "loss": 1.1182, + "margin_dpo/margin_mean": 470.66485595703125, + "margin_dpo/margin_std": 715.6707763671875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -805.451171875, + "KL/mean": -1021.7246704101562, + "KL/rejected_KL_mean": -1237.998291015625, + "KL/std": 608.487060546875, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0008413844625465572, + "fcm_dpo/delta": 0.0402926467359066, + "fcm_dpo/margin": 432.547119140625, + "fcm_dpo/q_t": 0.41544196009635925, + "grad_norm": 38.69491958618164, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.142184853553772, + "logits/rejected": -1.1601972579956055, + "logps/chosen": -869.6646728515625, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1329.65185546875, + "loss": 1.1185, + "margin_dpo/margin_mean": 432.547119140625, + "margin_dpo/margin_std": 615.1461181640625, + "step": 629 + }, + { + "KL/chosen_KL_mean": -716.5084838867188, + "KL/mean": -844.033447265625, + "KL/rejected_KL_mean": -971.5584106445312, + "KL/std": 613.792724609375, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0008560253772884607, + "fcm_dpo/delta": 0.046060267835855484, + "fcm_dpo/margin": 255.04989624023438, + "fcm_dpo/q_t": 0.45485490560531616, + "grad_norm": 46.711185455322266, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -1.1339406967163086, + "logits/rejected": -1.1089670658111572, + "logps/chosen": -775.7994995117188, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1055.15673828125, + "loss": 1.2745, + "margin_dpo/margin_mean": 255.0498809814453, + "margin_dpo/margin_std": 750.509033203125, + "step": 630 + }, + { + "KL/chosen_KL_mean": -839.9686279296875, + "KL/mean": -1040.942138671875, + "KL/rejected_KL_mean": -1241.91552734375, + "KL/std": 608.43701171875, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0008895150385797024, + "fcm_dpo/delta": 0.04728509113192558, + "fcm_dpo/margin": 401.9468688964844, + "fcm_dpo/q_t": 0.41873958706855774, + "grad_norm": 34.53041076660156, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.198280692100525, + "logits/rejected": -1.2126449346542358, + "logps/chosen": -899.4222412109375, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1322.8670654296875, + "loss": 1.1557, + "margin_dpo/margin_mean": 401.9468688964844, + "margin_dpo/margin_std": 713.8795166015625, + "step": 631 + }, + { + "KL/chosen_KL_mean": -755.68701171875, + "KL/mean": -961.2128295898438, + "KL/rejected_KL_mean": -1166.73876953125, + "KL/std": 571.27783203125, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.000894852913916111, + "fcm_dpo/delta": 0.03534339368343353, + "fcm_dpo/margin": 411.05169677734375, + "fcm_dpo/q_t": 0.41672447323799133, + "grad_norm": 49.45858383178711, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.1680915355682373, + "logits/rejected": -1.1694086790084839, + "logps/chosen": -817.03857421875, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1252.89892578125, + "loss": 1.1477, + "margin_dpo/margin_mean": 411.05169677734375, + "margin_dpo/margin_std": 707.6513061523438, + "step": 632 + }, + { + "KL/chosen_KL_mean": -829.6671142578125, + "KL/mean": -982.169921875, + "KL/rejected_KL_mean": -1134.672607421875, + "KL/std": 559.980712890625, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0009687743149697781, + "fcm_dpo/delta": 0.11273960769176483, + "fcm_dpo/margin": 305.00555419921875, + "fcm_dpo/q_t": 0.43423759937286377, + "grad_norm": 57.426727294921875, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.1930570602416992, + "logits/rejected": -1.2052525281906128, + "logps/chosen": -886.9453125, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1226.256591796875, + "loss": 1.2042, + "margin_dpo/margin_mean": 305.00555419921875, + "margin_dpo/margin_std": 643.3433837890625, + "step": 633 + }, + { + "KL/chosen_KL_mean": -920.3572998046875, + "KL/mean": -1058.0430908203125, + "KL/rejected_KL_mean": -1195.72900390625, + "KL/std": 668.88916015625, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0010300370631739497, + "fcm_dpo/delta": 0.12746718525886536, + "fcm_dpo/margin": 275.3716125488281, + "fcm_dpo/q_t": 0.44023919105529785, + "grad_norm": 39.72129821777344, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -1.0394319295883179, + "logits/rejected": -1.0358493328094482, + "logps/chosen": -986.9762573242188, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1302.8544921875, + "loss": 1.2264, + "margin_dpo/margin_mean": 275.37158203125, + "margin_dpo/margin_std": 653.4959716796875, + "step": 634 + }, + { + "KL/chosen_KL_mean": -813.5724487304688, + "KL/mean": -996.7451782226562, + "KL/rejected_KL_mean": -1179.9178466796875, + "KL/std": 701.66455078125, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.0010614688508212566, + "fcm_dpo/delta": 0.012546280398964882, + "fcm_dpo/margin": 366.345458984375, + "fcm_dpo/q_t": 0.41762009263038635, + "grad_norm": 67.32111358642578, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.135990858078003, + "logits/rejected": -1.1185557842254639, + "logps/chosen": -887.5279541015625, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1262.4183349609375, + "loss": 1.1567, + "margin_dpo/margin_mean": 366.345458984375, + "margin_dpo/margin_std": 705.2263793945312, + "step": 635 + }, + { + "KL/chosen_KL_mean": -745.6792602539062, + "KL/mean": -974.3742065429688, + "KL/rejected_KL_mean": -1203.0692138671875, + "KL/std": 660.8221435546875, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0010439862962812185, + "fcm_dpo/delta": -0.09222058206796646, + "fcm_dpo/margin": 457.389892578125, + "fcm_dpo/q_t": 0.3959377110004425, + "grad_norm": 29.998964309692383, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -1.056973934173584, + "logits/rejected": -1.0640333890914917, + "logps/chosen": -805.3081665039062, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1285.048095703125, + "loss": 1.0791, + "margin_dpo/margin_mean": 457.389892578125, + "margin_dpo/margin_std": 696.8487548828125, + "step": 636 + }, + { + "KL/chosen_KL_mean": -756.3297119140625, + "KL/mean": -1009.203369140625, + "KL/rejected_KL_mean": -1262.0771484375, + "KL/std": 668.9910278320312, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0009452562080696225, + "fcm_dpo/delta": -0.09541159868240356, + "fcm_dpo/margin": 505.7474670410156, + "fcm_dpo/q_t": 0.39195752143859863, + "grad_norm": 32.35881805419922, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.1782724857330322, + "logits/rejected": -1.2334554195404053, + "logps/chosen": -805.9823608398438, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1360.482177734375, + "loss": 1.0589, + "margin_dpo/margin_mean": 505.7474365234375, + "margin_dpo/margin_std": 686.821533203125, + "step": 637 + }, + { + "KL/chosen_KL_mean": -726.539794921875, + "KL/mean": -933.4669189453125, + "KL/rejected_KL_mean": -1140.394287109375, + "KL/std": 624.756591796875, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0009759259410202503, + "fcm_dpo/delta": -0.011953838169574738, + "fcm_dpo/margin": 413.85443115234375, + "fcm_dpo/q_t": 0.4031156003475189, + "grad_norm": 40.10390090942383, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -1.0225838422775269, + "logits/rejected": -1.0060951709747314, + "logps/chosen": -784.6964111328125, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1219.6956787109375, + "loss": 1.1516, + "margin_dpo/margin_mean": 413.85443115234375, + "margin_dpo/margin_std": 738.66943359375, + "step": 638 + }, + { + "KL/chosen_KL_mean": -1005.75341796875, + "KL/mean": -1090.333251953125, + "KL/rejected_KL_mean": -1174.9130859375, + "KL/std": 601.1937255859375, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0009415241074748337, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.15966796875, + "fcm_dpo/q_t": 0.4645419716835022, + "grad_norm": 118.68116760253906, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -1.0068409442901611, + "logits/rejected": -0.9428518414497375, + "logps/chosen": -1078.07666015625, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1249.18798828125, + "loss": 1.3509, + "margin_dpo/margin_mean": 169.15969848632812, + "margin_dpo/margin_std": 748.463623046875, + "step": 639 + }, + { + "KL/chosen_KL_mean": -750.6405029296875, + "KL/mean": -1046.021484375, + "KL/rejected_KL_mean": -1341.4024658203125, + "KL/std": 683.87060546875, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0008923726854845881, + "fcm_dpo/delta": -0.15029767155647278, + "fcm_dpo/margin": 590.761962890625, + "fcm_dpo/q_t": 0.3836818337440491, + "grad_norm": 49.275699615478516, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -1.0208163261413574, + "logits/rejected": -1.0585215091705322, + "logps/chosen": -806.7747802734375, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1450.0025634765625, + "loss": 1.04, + "margin_dpo/margin_mean": 590.761962890625, + "margin_dpo/margin_std": 807.516845703125, + "step": 640 + }, + { + "KL/chosen_KL_mean": -872.4000244140625, + "KL/mean": -1072.265380859375, + "KL/rejected_KL_mean": -1272.130859375, + "KL/std": 596.3260498046875, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0008794406894594431, + "fcm_dpo/delta": 0.05425509810447693, + "fcm_dpo/margin": 399.73089599609375, + "fcm_dpo/q_t": 0.4243098497390747, + "grad_norm": 28.6408748626709, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.1158493757247925, + "logits/rejected": -1.132592797279358, + "logps/chosen": -937.3968505859375, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1359.123291015625, + "loss": 1.1786, + "margin_dpo/margin_mean": 399.73089599609375, + "margin_dpo/margin_std": 807.8192138671875, + "step": 641 + }, + { + "KL/chosen_KL_mean": -836.827880859375, + "KL/mean": -1075.2613525390625, + "KL/rejected_KL_mean": -1313.69482421875, + "KL/std": 774.7606201171875, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0008894011261872947, + "fcm_dpo/delta": -0.028488531708717346, + "fcm_dpo/margin": 476.8669128417969, + "fcm_dpo/q_t": 0.41375476121902466, + "grad_norm": 37.86272430419922, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -1.0244096517562866, + "logits/rejected": -1.065093755722046, + "logps/chosen": -902.5171508789062, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1423.936767578125, + "loss": 1.1515, + "margin_dpo/margin_mean": 476.8669128417969, + "margin_dpo/margin_std": 952.716064453125, + "step": 642 + }, + { + "KL/chosen_KL_mean": -756.6348876953125, + "KL/mean": -941.3033447265625, + "KL/rejected_KL_mean": -1125.971923828125, + "KL/std": 589.2139282226562, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0008562486036680639, + "fcm_dpo/delta": -0.033700812608003616, + "fcm_dpo/margin": 369.3368835449219, + "fcm_dpo/q_t": 0.42599809169769287, + "grad_norm": 38.115516662597656, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.186774492263794, + "logits/rejected": -1.1872531175613403, + "logps/chosen": -808.5848999023438, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1213.440185546875, + "loss": 1.1603, + "margin_dpo/margin_mean": 369.3368835449219, + "margin_dpo/margin_std": 593.1607055664062, + "step": 643 + }, + { + "KL/chosen_KL_mean": -742.0232543945312, + "KL/mean": -933.4611206054688, + "KL/rejected_KL_mean": -1124.89892578125, + "KL/std": 665.626220703125, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0008826255798339844, + "fcm_dpo/delta": 0.0690830871462822, + "fcm_dpo/margin": 382.8756103515625, + "fcm_dpo/q_t": 0.4265103042125702, + "grad_norm": 49.10441589355469, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.072190761566162, + "logits/rejected": -1.085551142692566, + "logps/chosen": -801.0408935546875, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1212.03564453125, + "loss": 1.186, + "margin_dpo/margin_mean": 382.8756103515625, + "margin_dpo/margin_std": 778.0350341796875, + "step": 644 + }, + { + "KL/chosen_KL_mean": -773.3355712890625, + "KL/mean": -864.772705078125, + "KL/rejected_KL_mean": -956.2098388671875, + "KL/std": 525.473876953125, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0009301776299253106, + "fcm_dpo/delta": 0.06894159317016602, + "fcm_dpo/margin": 182.87423706054688, + "fcm_dpo/q_t": 0.46347182989120483, + "grad_norm": 111.940673828125, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -1.0397567749023438, + "logits/rejected": -1.0286939144134521, + "logps/chosen": -829.2116088867188, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1053.99072265625, + "loss": 1.335, + "margin_dpo/margin_mean": 182.87423706054688, + "margin_dpo/margin_std": 713.6385498046875, + "step": 645 + }, + { + "KL/chosen_KL_mean": -730.0833129882812, + "KL/mean": -890.043212890625, + "KL/rejected_KL_mean": -1050.0030517578125, + "KL/std": 501.4505615234375, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0009639813215471804, + "fcm_dpo/delta": 0.10127154737710953, + "fcm_dpo/margin": 319.91973876953125, + "fcm_dpo/q_t": 0.43163058161735535, + "grad_norm": 59.23325729370117, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.0868816375732422, + "logits/rejected": -1.0689315795898438, + "logps/chosen": -791.359130859375, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1127.5087890625, + "loss": 1.1827, + "margin_dpo/margin_mean": 319.91973876953125, + "margin_dpo/margin_std": 596.3582153320312, + "step": 646 + }, + { + "KL/chosen_KL_mean": -651.00048828125, + "KL/mean": -853.3546142578125, + "KL/rejected_KL_mean": -1055.708740234375, + "KL/std": 576.6222534179688, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0009885020554065704, + "fcm_dpo/delta": -6.2420847825706e-05, + "fcm_dpo/margin": 404.70819091796875, + "fcm_dpo/q_t": 0.40860414505004883, + "grad_norm": 38.0550537109375, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.9868925213813782, + "logits/rejected": -1.020545482635498, + "logps/chosen": -705.8529052734375, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1149.2281494140625, + "loss": 1.1061, + "margin_dpo/margin_mean": 404.70819091796875, + "margin_dpo/margin_std": 587.0342407226562, + "step": 647 + }, + { + "KL/chosen_KL_mean": -685.2008056640625, + "KL/mean": -991.873779296875, + "KL/rejected_KL_mean": -1298.5467529296875, + "KL/std": 712.3465576171875, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0009071773965843022, + "fcm_dpo/delta": -0.1885771006345749, + "fcm_dpo/margin": 613.345947265625, + "fcm_dpo/q_t": 0.37869709730148315, + "grad_norm": 30.786937713623047, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.1172411441802979, + "logits/rejected": -1.183282732963562, + "logps/chosen": -739.3722534179688, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1397.259521484375, + "loss": 1.0291, + "margin_dpo/margin_mean": 613.345947265625, + "margin_dpo/margin_std": 863.4343872070312, + "step": 648 + }, + { + "KL/chosen_KL_mean": -726.219970703125, + "KL/mean": -859.9382934570312, + "KL/rejected_KL_mean": -993.6566772460938, + "KL/std": 530.8839721679688, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0008987129549495876, + "fcm_dpo/delta": 0.059498947113752365, + "fcm_dpo/margin": 267.4367370605469, + "fcm_dpo/q_t": 0.44877535104751587, + "grad_norm": 33.56425476074219, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.1480650901794434, + "logits/rejected": -1.1410375833511353, + "logps/chosen": -788.7003173828125, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1073.73388671875, + "loss": 1.2488, + "margin_dpo/margin_mean": 267.4367370605469, + "margin_dpo/margin_std": 664.775390625, + "step": 649 + }, + { + "KL/chosen_KL_mean": -780.0118408203125, + "KL/mean": -1001.0939331054688, + "KL/rejected_KL_mean": -1222.176025390625, + "KL/std": 675.9951171875, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0009087229846045375, + "fcm_dpo/delta": -0.0022899359464645386, + "fcm_dpo/margin": 442.1641845703125, + "fcm_dpo/q_t": 0.411517858505249, + "grad_norm": 33.60219192504883, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.0947818756103516, + "logits/rejected": -1.119983434677124, + "logps/chosen": -836.1046142578125, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1320.44091796875, + "loss": 1.1362, + "margin_dpo/margin_mean": 442.16424560546875, + "margin_dpo/margin_std": 771.5479736328125, + "step": 650 + }, + { + "KL/chosen_KL_mean": -523.4451904296875, + "KL/mean": -814.3759155273438, + "KL/rejected_KL_mean": -1105.306640625, + "KL/std": 615.1279907226562, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0008620844455435872, + "fcm_dpo/delta": -0.12045219540596008, + "fcm_dpo/margin": 581.8615112304688, + "fcm_dpo/q_t": 0.3833530843257904, + "grad_norm": 38.2287712097168, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.112557053565979, + "logits/rejected": -1.177018642425537, + "logps/chosen": -566.87060546875, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1205.2646484375, + "loss": 1.014, + "margin_dpo/margin_mean": 581.861572265625, + "margin_dpo/margin_std": 639.6220092773438, + "step": 651 + }, + { + "KL/chosen_KL_mean": -658.0653076171875, + "KL/mean": -875.8676147460938, + "KL/rejected_KL_mean": -1093.669921875, + "KL/std": 614.32666015625, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0008646683418191969, + "fcm_dpo/delta": 0.02592024952173233, + "fcm_dpo/margin": 435.6046142578125, + "fcm_dpo/q_t": 0.4156026840209961, + "grad_norm": 31.640792846679688, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.0805503129959106, + "logits/rejected": -1.1101926565170288, + "logps/chosen": -720.64208984375, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1205.437744140625, + "loss": 1.1279, + "margin_dpo/margin_mean": 435.6046142578125, + "margin_dpo/margin_std": 698.932373046875, + "step": 652 + }, + { + "KL/chosen_KL_mean": -797.345947265625, + "KL/mean": -1008.8164672851562, + "KL/rejected_KL_mean": -1220.286865234375, + "KL/std": 670.9261474609375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0008779562194831669, + "fcm_dpo/delta": 0.032121218740940094, + "fcm_dpo/margin": 422.94091796875, + "fcm_dpo/q_t": 0.4186922609806061, + "grad_norm": 35.974857330322266, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.1680411100387573, + "logits/rejected": -1.1997288465499878, + "logps/chosen": -858.458984375, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1323.53662109375, + "loss": 1.1385, + "margin_dpo/margin_mean": 422.94091796875, + "margin_dpo/margin_std": 715.555419921875, + "step": 653 + }, + { + "KL/chosen_KL_mean": -730.664794921875, + "KL/mean": -928.2381591796875, + "KL/rejected_KL_mean": -1125.8115234375, + "KL/std": 545.0596923828125, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0008997832774184644, + "fcm_dpo/delta": 0.04974420368671417, + "fcm_dpo/margin": 395.1466979980469, + "fcm_dpo/q_t": 0.42145484685897827, + "grad_norm": 32.788177490234375, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.1008021831512451, + "logits/rejected": -1.1207127571105957, + "logps/chosen": -792.3929443359375, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1224.5853271484375, + "loss": 1.1387, + "margin_dpo/margin_mean": 395.1466979980469, + "margin_dpo/margin_std": 639.1888427734375, + "step": 654 + }, + { + "KL/chosen_KL_mean": -684.8628540039062, + "KL/mean": -883.15283203125, + "KL/rejected_KL_mean": -1081.4427490234375, + "KL/std": 549.9573974609375, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0009232647716999054, + "fcm_dpo/delta": 0.03795754536986351, + "fcm_dpo/margin": 396.5799255371094, + "fcm_dpo/q_t": 0.41847848892211914, + "grad_norm": 32.03719711303711, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.095155954360962, + "logits/rejected": -1.1428718566894531, + "logps/chosen": -734.439697265625, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1179.734619140625, + "loss": 1.1367, + "margin_dpo/margin_mean": 396.5799255371094, + "margin_dpo/margin_std": 650.6702880859375, + "step": 655 + }, + { + "KL/chosen_KL_mean": -729.98681640625, + "KL/mean": -1082.38134765625, + "KL/rejected_KL_mean": -1434.77587890625, + "KL/std": 763.218017578125, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0008440595702268183, + "fcm_dpo/delta": -0.2358550727367401, + "fcm_dpo/margin": 704.7891845703125, + "fcm_dpo/q_t": 0.36440345644950867, + "grad_norm": 68.34577941894531, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.9933082461357117, + "logits/rejected": -1.0727477073669434, + "logps/chosen": -782.5362548828125, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1548.45068359375, + "loss": 0.9698, + "margin_dpo/margin_mean": 704.7891845703125, + "margin_dpo/margin_std": 770.411376953125, + "step": 656 + }, + { + "KL/chosen_KL_mean": -678.0421142578125, + "KL/mean": -969.9865112304688, + "KL/rejected_KL_mean": -1261.930908203125, + "KL/std": 704.830322265625, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0007784855552017689, + "fcm_dpo/delta": -0.06374643743038177, + "fcm_dpo/margin": 583.888916015625, + "fcm_dpo/q_t": 0.39540228247642517, + "grad_norm": 39.44537353515625, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.103132724761963, + "logits/rejected": -1.1629598140716553, + "logps/chosen": -724.7426147460938, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1359.8458251953125, + "loss": 1.0552, + "margin_dpo/margin_mean": 583.888916015625, + "margin_dpo/margin_std": 730.0590209960938, + "step": 657 + }, + { + "KL/chosen_KL_mean": -735.4950561523438, + "KL/mean": -952.8492431640625, + "KL/rejected_KL_mean": -1170.20361328125, + "KL/std": 534.92578125, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0007870746776461601, + "fcm_dpo/delta": 0.06411394476890564, + "fcm_dpo/margin": 434.7084655761719, + "fcm_dpo/q_t": 0.4210846424102783, + "grad_norm": 32.0853271484375, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -1.0991567373275757, + "logits/rejected": -1.133607268333435, + "logps/chosen": -796.4532470703125, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1266.14306640625, + "loss": 1.1304, + "margin_dpo/margin_mean": 434.70849609375, + "margin_dpo/margin_std": 604.7078857421875, + "step": 658 + }, + { + "KL/chosen_KL_mean": -640.479248046875, + "KL/mean": -857.1295166015625, + "KL/rejected_KL_mean": -1073.77978515625, + "KL/std": 558.2611083984375, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0008168879430741072, + "fcm_dpo/delta": 0.051437366753816605, + "fcm_dpo/margin": 433.300537109375, + "fcm_dpo/q_t": 0.4200027287006378, + "grad_norm": 38.316715240478516, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -1.0011135339736938, + "logits/rejected": -0.9742828607559204, + "logps/chosen": -717.2222900390625, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1161.250732421875, + "loss": 1.1219, + "margin_dpo/margin_mean": 433.3005065917969, + "margin_dpo/margin_std": 611.8961181640625, + "step": 659 + }, + { + "KL/chosen_KL_mean": -718.798828125, + "KL/mean": -990.9500122070312, + "KL/rejected_KL_mean": -1263.10107421875, + "KL/std": 653.447021484375, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0008152863010764122, + "fcm_dpo/delta": -0.05039425566792488, + "fcm_dpo/margin": 544.3023071289062, + "fcm_dpo/q_t": 0.39815008640289307, + "grad_norm": 36.57526779174805, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.1413556337356567, + "logits/rejected": -1.160224437713623, + "logps/chosen": -777.8467407226562, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1339.0611572265625, + "loss": 1.067, + "margin_dpo/margin_mean": 544.30224609375, + "margin_dpo/margin_std": 707.3837280273438, + "step": 660 + }, + { + "KL/chosen_KL_mean": -626.880126953125, + "KL/mean": -879.5157470703125, + "KL/rejected_KL_mean": -1132.1513671875, + "KL/std": 713.3192138671875, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0008107817848213017, + "fcm_dpo/delta": -0.011798802763223648, + "fcm_dpo/margin": 505.2711486816406, + "fcm_dpo/q_t": 0.4084544777870178, + "grad_norm": 40.93271255493164, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.1234712600708008, + "logits/rejected": -1.1440787315368652, + "logps/chosen": -677.5540771484375, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1218.156982421875, + "loss": 1.0911, + "margin_dpo/margin_mean": 505.2711486816406, + "margin_dpo/margin_std": 714.84130859375, + "step": 661 + }, + { + "KL/chosen_KL_mean": -737.6925048828125, + "KL/mean": -933.1785888671875, + "KL/rejected_KL_mean": -1128.664794921875, + "KL/std": 589.3681640625, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0008305037626996636, + "fcm_dpo/delta": 0.08339127153158188, + "fcm_dpo/margin": 390.97222900390625, + "fcm_dpo/q_t": 0.4276695251464844, + "grad_norm": 31.396419525146484, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -1.0537675619125366, + "logits/rejected": -1.0568206310272217, + "logps/chosen": -806.9534912109375, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1217.720703125, + "loss": 1.1746, + "margin_dpo/margin_mean": 390.97222900390625, + "margin_dpo/margin_std": 735.9049072265625, + "step": 662 + }, + { + "KL/chosen_KL_mean": -703.304931640625, + "KL/mean": -951.3535766601562, + "KL/rejected_KL_mean": -1199.40234375, + "KL/std": 692.343017578125, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0008315940503962338, + "fcm_dpo/delta": -0.01457483321428299, + "fcm_dpo/margin": 496.09747314453125, + "fcm_dpo/q_t": 0.40950340032577515, + "grad_norm": 29.27633285522461, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.108817458152771, + "logits/rejected": -1.1424871683120728, + "logps/chosen": -768.183837890625, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1313.32763671875, + "loss": 1.1149, + "margin_dpo/margin_mean": 496.09747314453125, + "margin_dpo/margin_std": 815.9259643554688, + "step": 663 + }, + { + "KL/chosen_KL_mean": -720.2525634765625, + "KL/mean": -981.3237915039062, + "KL/rejected_KL_mean": -1242.39501953125, + "KL/std": 640.775634765625, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0008131160284392536, + "fcm_dpo/delta": -0.029562106356024742, + "fcm_dpo/margin": 522.1424560546875, + "fcm_dpo/q_t": 0.4030224680900574, + "grad_norm": 30.475723266601562, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.083822250366211, + "logits/rejected": -1.1255340576171875, + "logps/chosen": -781.1409912109375, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1347.916748046875, + "loss": 1.0806, + "margin_dpo/margin_mean": 522.1424560546875, + "margin_dpo/margin_std": 695.9418334960938, + "step": 664 + }, + { + "KL/chosen_KL_mean": -647.258544921875, + "KL/mean": -869.3076171875, + "KL/rejected_KL_mean": -1091.3568115234375, + "KL/std": 543.1394653320312, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0008315135492011905, + "fcm_dpo/delta": 0.03456343337893486, + "fcm_dpo/margin": 444.0981750488281, + "fcm_dpo/q_t": 0.41486600041389465, + "grad_norm": 50.45151138305664, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.1923636198043823, + "logits/rejected": -1.2248049974441528, + "logps/chosen": -707.8226928710938, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1176.16552734375, + "loss": 1.1035, + "margin_dpo/margin_mean": 444.0981750488281, + "margin_dpo/margin_std": 563.846923828125, + "step": 665 + }, + { + "KL/chosen_KL_mean": -706.4842529296875, + "KL/mean": -933.85595703125, + "KL/rejected_KL_mean": -1161.2275390625, + "KL/std": 587.65478515625, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0008393836324103177, + "fcm_dpo/delta": 0.020563848316669464, + "fcm_dpo/margin": 454.74334716796875, + "fcm_dpo/q_t": 0.41398605704307556, + "grad_norm": 32.466407775878906, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -1.002959132194519, + "logits/rejected": -1.0180656909942627, + "logps/chosen": -770.9042358398438, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1257.1192626953125, + "loss": 1.1149, + "margin_dpo/margin_mean": 454.7433776855469, + "margin_dpo/margin_std": 661.7647705078125, + "step": 666 + }, + { + "KL/chosen_KL_mean": -800.3102416992188, + "KL/mean": -1067.533203125, + "KL/rejected_KL_mean": -1334.75634765625, + "KL/std": 639.68212890625, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.00082051643403247, + "fcm_dpo/delta": -0.04578736796975136, + "fcm_dpo/margin": 534.446044921875, + "fcm_dpo/q_t": 0.39925211668014526, + "grad_norm": 32.55545425415039, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -1.0411145687103271, + "logits/rejected": -1.0438776016235352, + "logps/chosen": -869.5872802734375, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1422.591796875, + "loss": 1.0738, + "margin_dpo/margin_mean": 534.446044921875, + "margin_dpo/margin_std": 710.6553955078125, + "step": 667 + }, + { + "KL/chosen_KL_mean": -832.54736328125, + "KL/mean": -1023.1879272460938, + "KL/rejected_KL_mean": -1213.8284912109375, + "KL/std": 671.5996704101562, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0008637277642264962, + "fcm_dpo/delta": 0.076044961810112, + "fcm_dpo/margin": 381.2811279296875, + "fcm_dpo/q_t": 0.42992591857910156, + "grad_norm": 39.28108215332031, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.0831975936889648, + "logits/rejected": -1.0906332731246948, + "logps/chosen": -905.1513671875, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1317.567626953125, + "loss": 1.1938, + "margin_dpo/margin_mean": 381.2811584472656, + "margin_dpo/margin_std": 802.906982421875, + "step": 668 + }, + { + "KL/chosen_KL_mean": -635.2991333007812, + "KL/mean": -900.4949340820312, + "KL/rejected_KL_mean": -1165.690673828125, + "KL/std": 621.0604858398438, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0008439387893304229, + "fcm_dpo/delta": -0.05490920692682266, + "fcm_dpo/margin": 530.3916015625, + "fcm_dpo/q_t": 0.3967704176902771, + "grad_norm": 27.538616180419922, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.036536455154419, + "logits/rejected": -1.0469627380371094, + "logps/chosen": -681.41552734375, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1243.6151123046875, + "loss": 1.0694, + "margin_dpo/margin_mean": 530.3916015625, + "margin_dpo/margin_std": 704.3974609375, + "step": 669 + }, + { + "KL/chosen_KL_mean": -614.975830078125, + "KL/mean": -878.3056030273438, + "KL/rejected_KL_mean": -1141.63525390625, + "KL/std": 589.9856567382812, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0008297088206745684, + "fcm_dpo/delta": -0.04324834421277046, + "fcm_dpo/margin": 526.6594848632812, + "fcm_dpo/q_t": 0.3996073007583618, + "grad_norm": 25.988235473632812, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.9836372137069702, + "logits/rejected": -1.0047441720962524, + "logps/chosen": -677.3216552734375, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1238.575927734375, + "loss": 1.0792, + "margin_dpo/margin_mean": 526.659423828125, + "margin_dpo/margin_std": 708.1810913085938, + "step": 670 + }, + { + "KL/chosen_KL_mean": -776.1077880859375, + "KL/mean": -988.3963623046875, + "KL/rejected_KL_mean": -1200.684814453125, + "KL/std": 598.3868408203125, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.00083927036030218, + "fcm_dpo/delta": 0.04752415046095848, + "fcm_dpo/margin": 424.57708740234375, + "fcm_dpo/q_t": 0.4184950292110443, + "grad_norm": 40.60873031616211, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.1314098834991455, + "logits/rejected": -1.1654772758483887, + "logps/chosen": -824.10791015625, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1284.504150390625, + "loss": 1.1507, + "margin_dpo/margin_mean": 424.5771179199219, + "margin_dpo/margin_std": 733.5236206054688, + "step": 671 + }, + { + "KL/chosen_KL_mean": -856.5309448242188, + "KL/mean": -1074.56787109375, + "KL/rejected_KL_mean": -1292.604736328125, + "KL/std": 727.41259765625, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0008428269065916538, + "fcm_dpo/delta": 0.03628811612725258, + "fcm_dpo/margin": 436.07379150390625, + "fcm_dpo/q_t": 0.41948267817497253, + "grad_norm": 39.966148376464844, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.191899299621582, + "logits/rejected": -1.2052488327026367, + "logps/chosen": -915.1142578125, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1385.744873046875, + "loss": 1.1611, + "margin_dpo/margin_mean": 436.07379150390625, + "margin_dpo/margin_std": 806.387939453125, + "step": 672 + }, + { + "KL/chosen_KL_mean": -779.7969970703125, + "KL/mean": -983.0469970703125, + "KL/rejected_KL_mean": -1186.296875, + "KL/std": 593.0384521484375, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0008732416899874806, + "fcm_dpo/delta": 0.05043650418519974, + "fcm_dpo/margin": 406.49993896484375, + "fcm_dpo/q_t": 0.420853853225708, + "grad_norm": 40.34420394897461, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.1462502479553223, + "logits/rejected": -1.1522610187530518, + "logps/chosen": -826.5201416015625, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1271.5931396484375, + "loss": 1.1355, + "margin_dpo/margin_mean": 406.49993896484375, + "margin_dpo/margin_std": 656.7118530273438, + "step": 673 + }, + { + "KL/chosen_KL_mean": -648.8712768554688, + "KL/mean": -916.4680786132812, + "KL/rejected_KL_mean": -1184.0648193359375, + "KL/std": 593.7801513671875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0008612494566477835, + "fcm_dpo/delta": -0.07060980051755905, + "fcm_dpo/margin": 535.1935424804688, + "fcm_dpo/q_t": 0.3969615399837494, + "grad_norm": 34.42951202392578, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -1.0662541389465332, + "logits/rejected": -1.0901139974594116, + "logps/chosen": -694.3168334960938, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1254.11083984375, + "loss": 1.0641, + "margin_dpo/margin_mean": 535.193603515625, + "margin_dpo/margin_std": 719.1665649414062, + "step": 674 + }, + { + "KL/chosen_KL_mean": -705.6675415039062, + "KL/mean": -978.6163940429688, + "KL/rejected_KL_mean": -1251.565185546875, + "KL/std": 659.03173828125, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0008071311167441308, + "fcm_dpo/delta": -0.05166063457727432, + "fcm_dpo/margin": 545.8975830078125, + "fcm_dpo/q_t": 0.39999037981033325, + "grad_norm": 29.014299392700195, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.0835530757904053, + "logits/rejected": -1.1036348342895508, + "logps/chosen": -749.8438720703125, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1325.6572265625, + "loss": 1.07, + "margin_dpo/margin_mean": 545.8975830078125, + "margin_dpo/margin_std": 693.9437255859375, + "step": 675 + }, + { + "KL/chosen_KL_mean": -790.8041381835938, + "KL/mean": -1041.859375, + "KL/rejected_KL_mean": -1292.91455078125, + "KL/std": 625.9526977539062, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0008189495420083404, + "fcm_dpo/delta": -0.01295204646885395, + "fcm_dpo/margin": 502.1104736328125, + "fcm_dpo/q_t": 0.40602952241897583, + "grad_norm": 26.191553115844727, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -1.047107458114624, + "logits/rejected": -1.0656406879425049, + "logps/chosen": -862.20263671875, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1381.2733154296875, + "loss": 1.0849, + "margin_dpo/margin_mean": 502.1104736328125, + "margin_dpo/margin_std": 653.969482421875, + "step": 676 + }, + { + "KL/chosen_KL_mean": -753.7366943359375, + "KL/mean": -1007.7711181640625, + "KL/rejected_KL_mean": -1261.8055419921875, + "KL/std": 641.484619140625, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0008058074745349586, + "fcm_dpo/delta": -0.010841760784387589, + "fcm_dpo/margin": 508.0688171386719, + "fcm_dpo/q_t": 0.4106763005256653, + "grad_norm": 27.45729637145996, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.134361743927002, + "logits/rejected": -1.133501648902893, + "logps/chosen": -810.26416015625, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1340.031982421875, + "loss": 1.1025, + "margin_dpo/margin_mean": 508.0688171386719, + "margin_dpo/margin_std": 777.37109375, + "step": 677 + }, + { + "KL/chosen_KL_mean": -652.7081298828125, + "KL/mean": -947.4844970703125, + "KL/rejected_KL_mean": -1242.2607421875, + "KL/std": 714.5780029296875, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0007915610331110656, + "fcm_dpo/delta": -0.07787713408470154, + "fcm_dpo/margin": 589.5526123046875, + "fcm_dpo/q_t": 0.39380910992622375, + "grad_norm": 31.252840042114258, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -1.0775364637374878, + "logits/rejected": -1.1228598356246948, + "logps/chosen": -698.8426513671875, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1322.865478515625, + "loss": 1.0638, + "margin_dpo/margin_mean": 589.552490234375, + "margin_dpo/margin_std": 798.2293701171875, + "step": 678 + }, + { + "KL/chosen_KL_mean": -742.26318359375, + "KL/mean": -949.9349975585938, + "KL/rejected_KL_mean": -1157.60693359375, + "KL/std": 570.056640625, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.000795794534496963, + "fcm_dpo/delta": 0.0772286057472229, + "fcm_dpo/margin": 415.34356689453125, + "fcm_dpo/q_t": 0.4261971712112427, + "grad_norm": 32.16876983642578, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -1.0969929695129395, + "logits/rejected": -1.0927238464355469, + "logps/chosen": -792.55810546875, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1234.2049560546875, + "loss": 1.1502, + "margin_dpo/margin_mean": 415.34356689453125, + "margin_dpo/margin_std": 676.8760986328125, + "step": 679 + }, + { + "KL/chosen_KL_mean": -755.0833740234375, + "KL/mean": -1031.6068115234375, + "KL/rejected_KL_mean": -1308.130126953125, + "KL/std": 723.09326171875, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0007822597399353981, + "fcm_dpo/delta": -0.03936435282230377, + "fcm_dpo/margin": 553.046875, + "fcm_dpo/q_t": 0.4001197814941406, + "grad_norm": 38.99601364135742, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -1.0872653722763062, + "logits/rejected": -1.1259450912475586, + "logps/chosen": -831.9990234375, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1420.514892578125, + "loss": 1.0895, + "margin_dpo/margin_mean": 553.0468139648438, + "margin_dpo/margin_std": 794.7000732421875, + "step": 680 + }, + { + "KL/chosen_KL_mean": -758.192626953125, + "KL/mean": -973.5965576171875, + "KL/rejected_KL_mean": -1189.00048828125, + "KL/std": 620.39990234375, + "epoch": 1.0, + "fcm_dpo/beta": 0.0007777622085995972, + "fcm_dpo/delta": -0.04388820007443428, + "fcm_dpo/margin": 430.8079528808594, + "fcm_dpo/q_t": 0.42303466796875, + "grad_norm": 24.369367599487305, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.162775993347168, + "logits/rejected": -1.1861462593078613, + "logps/chosen": -819.14990234375, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1277.55859375, + "loss": 1.1537, + "margin_dpo/margin_mean": 430.80792236328125, + "margin_dpo/margin_std": 687.1182861328125, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.1186340892367426, + "train_runtime": 1737.4974, + "train_samples_per_second": 25.092, + "train_steps_per_second": 0.392 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}