From acb095f22ecb0bba2bc4b6ef0e65f80788da4c3e Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 1 Jun 2026 08:08:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..c510fdf --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..ca99cc2 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 0.9659692127799148, + "train_runtime": 1735.4678, + "train_samples": 43598, + "train_samples_per_second": 25.122, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..374171f --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f65d4595f2070d6d093ca81dc4d7205d9f046c7fb5fd8e92829d5f2e050ece85 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..233464d --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b24d6904e8ffc0b71b55f0b53499e26871aaf82ee6a6b1f81be885ba9037beb +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..63819ab --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5428599d5ae4cd589564b2326c385de14014203add153dd9d6f86fb56bbd687e +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..de17952 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e59246e91e1478c38d0dd1c7f21db893cea737925a5a8ed2c59b53ac7e8db602 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..232c767 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa60eee3c949cdea7e7aa94fe053506fe555af3c89a506d0053e4e0baa7bc8b4 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..67f5791 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cd410a789b2054a2a621e2ab48c83e5a6eee9b0aa4de31decc40549b5e04c9c +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..346bab4 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77ac09f5d7690cf8ac4ab02cf429a3fba81b21bb1c97fcb3bbbb23cabafd28f5 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..ca99cc2 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 0.9659692127799148, + "train_runtime": 1735.4678, + "train_samples": 43598, + "train_samples_per_second": 25.122, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..33fca3a --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.5112075805664, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.20995330810547, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.0140533447265625, + "KL/mean": -0.0008638352155685425, + "KL/rejected_KL_mean": 0.012327194213867188, + "KL/std": 0.22636425495147705, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02638193964958191, + "fcm_dpo/q_t": 0.5006595253944397, + "grad_norm": 71.01204681396484, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4816300868988037, + "logits/rejected": -0.4421873092651367, + "logps/chosen": -60.99565124511719, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.66027069091797, + "loss": 1.3892, + "margin_dpo/margin_mean": -0.02638128399848938, + "margin_dpo/margin_std": 0.3333635926246643, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.012470245361328125, + "KL/mean": 0.010987430810928345, + "KL/rejected_KL_mean": 0.009502410888671875, + "KL/std": 0.23756694793701172, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0029723048210144043, + "fcm_dpo/q_t": 0.4999255836009979, + "grad_norm": 71.9110107421875, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.46880286931991577, + "logits/rejected": -0.4411168396472931, + "logps/chosen": -56.75524139404297, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.63760375976562, + "loss": 1.3863, + "margin_dpo/margin_mean": 0.002971738576889038, + "margin_dpo/margin_std": 0.36297130584716797, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.0008716583251953125, + "KL/mean": 0.015165463089942932, + "KL/rejected_KL_mean": 0.0312042236328125, + "KL/std": 0.25330108404159546, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03207537531852722, + "fcm_dpo/q_t": 0.500801682472229, + "grad_norm": 90.06421661376953, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5142922401428223, + "logits/rejected": -0.47061434388160706, + "logps/chosen": -53.86024475097656, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.11798095703125, + "loss": 1.3897, + "margin_dpo/margin_mean": -0.032075434923172, + "margin_dpo/margin_std": 0.30366232991218567, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.033855438232421875, + "KL/mean": 0.0072780996561050415, + "KL/rejected_KL_mean": 0.048412322998046875, + "KL/std": 0.2958972454071045, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0822724997997284, + "fcm_dpo/q_t": 0.5020565986633301, + "grad_norm": 92.30109405517578, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.512769877910614, + "logits/rejected": -0.4712473750114441, + "logps/chosen": -63.04133987426758, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.59693908691406, + "loss": 1.395, + "margin_dpo/margin_mean": -0.08227315545082092, + "margin_dpo/margin_std": 0.41161593794822693, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.006443023681640625, + "KL/mean": 0.014417484402656555, + "KL/rejected_KL_mean": 0.02239227294921875, + "KL/std": 0.30512264370918274, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01594802737236023, + "fcm_dpo/q_t": 0.5003977417945862, + "grad_norm": 82.46339416503906, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5128216743469238, + "logits/rejected": -0.47885674238204956, + "logps/chosen": -57.76837921142578, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.89820098876953, + "loss": 1.3884, + "margin_dpo/margin_mean": -0.015947014093399048, + "margin_dpo/margin_std": 0.4566078186035156, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.0637359619140625, + "KL/mean": 0.06535476446151733, + "KL/rejected_KL_mean": 0.06697463989257812, + "KL/std": 0.2852301299571991, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0032398104667663574, + "fcm_dpo/q_t": 0.5000797510147095, + "grad_norm": 78.44632720947266, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5076254606246948, + "logits/rejected": -0.48168283700942993, + "logps/chosen": -58.65229797363281, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.24444580078125, + "loss": 1.387, + "margin_dpo/margin_mean": -0.0032394826412200928, + "margin_dpo/margin_std": 0.38733720779418945, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.01503753662109375, + "KL/mean": -0.0038666129112243652, + "KL/rejected_KL_mean": -0.02276611328125, + "KL/std": 0.2756863534450531, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03780454397201538, + "fcm_dpo/q_t": 0.4990554451942444, + "grad_norm": 85.13935089111328, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.493435800075531, + "logits/rejected": -0.44671258330345154, + "logps/chosen": -69.851806640625, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.62542724609375, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.03780451416969299, + "margin_dpo/margin_std": 0.37339428067207336, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.0016689300537109375, + "KL/mean": -0.0027171149849891663, + "KL/rejected_KL_mean": -0.00376129150390625, + "KL/std": 0.2760874032974243, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.002091839909553528, + "fcm_dpo/q_t": 0.4999482035636902, + "grad_norm": 70.59938049316406, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.47122734785079956, + "logits/rejected": -0.42694908380508423, + "logps/chosen": -48.359352111816406, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.37582397460938, + "loss": 1.3865, + "margin_dpo/margin_mean": 0.002091914415359497, + "margin_dpo/margin_std": 0.38940873742103577, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.04403114318847656, + "KL/mean": 0.002697855234146118, + "KL/rejected_KL_mean": -0.038639068603515625, + "KL/std": 0.24503415822982788, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08266651630401611, + "fcm_dpo/q_t": 0.49793386459350586, + "grad_norm": 68.08948516845703, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.4868287742137909, + "logits/rejected": -0.4630933403968811, + "logps/chosen": -52.97282409667969, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.81901550292969, + "loss": 1.3783, + "margin_dpo/margin_mean": 0.08266735076904297, + "margin_dpo/margin_std": 0.30503180623054504, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.04669380187988281, + "KL/mean": -0.03728406876325607, + "KL/rejected_KL_mean": -0.027866363525390625, + "KL/std": 0.30298733711242676, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.018821552395820618, + "fcm_dpo/q_t": 0.5004702210426331, + "grad_norm": 91.94474792480469, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5286523103713989, + "logits/rejected": -0.49216967821121216, + "logps/chosen": -61.85213088989258, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.88613891601562, + "loss": 1.3887, + "margin_dpo/margin_mean": -0.018821612000465393, + "margin_dpo/margin_std": 0.4304816722869873, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.03151130676269531, + "KL/mean": 0.02703501284122467, + "KL/rejected_KL_mean": 0.02256011962890625, + "KL/std": 0.25951892137527466, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.008960098028182983, + "fcm_dpo/q_t": 0.4997759163379669, + "grad_norm": 79.31588745117188, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.4967392683029175, + "logits/rejected": -0.46991458535194397, + "logps/chosen": -64.22884368896484, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.18052673339844, + "loss": 1.3858, + "margin_dpo/margin_mean": 0.008959829807281494, + "margin_dpo/margin_std": 0.38706913590431213, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.005828857421875, + "KL/mean": 0.0027678310871124268, + "KL/rejected_KL_mean": 0.01136016845703125, + "KL/std": 0.2991213798522949, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01718384027481079, + "fcm_dpo/q_t": 0.5004267692565918, + "grad_norm": 85.39857482910156, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.4677683115005493, + "logits/rejected": -0.4280736446380615, + "logps/chosen": -58.11603927612305, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.03572082519531, + "loss": 1.3886, + "margin_dpo/margin_mean": -0.017183274030685425, + "margin_dpo/margin_std": 0.4665626883506775, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.049083709716796875, + "KL/mean": -0.050711825489997864, + "KL/rejected_KL_mean": -0.052337646484375, + "KL/std": 0.23152416944503784, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0032589733600616455, + "fcm_dpo/q_t": 0.49991798400878906, + "grad_norm": 64.03571319580078, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.49719661474227905, + "logits/rejected": -0.478732705116272, + "logps/chosen": -57.0159912109375, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.86097717285156, + "loss": 1.3863, + "margin_dpo/margin_mean": 0.0032590627670288086, + "margin_dpo/margin_std": 0.33922165632247925, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.04007148742675781, + "KL/mean": 7.331371307373047e-06, + "KL/rejected_KL_mean": -0.0400543212890625, + "KL/std": 0.2565869390964508, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08013355731964111, + "fcm_dpo/q_t": 0.4979970455169678, + "grad_norm": 84.36243438720703, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5266684293746948, + "logits/rejected": -0.485817015171051, + "logps/chosen": -61.69982147216797, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.40953063964844, + "loss": 1.3786, + "margin_dpo/margin_mean": 0.08013346791267395, + "margin_dpo/margin_std": 0.33502477407455444, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.07616043090820312, + "KL/mean": -0.011057019233703613, + "KL/rejected_KL_mean": -0.09827423095703125, + "KL/std": 0.24816861748695374, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17442834377288818, + "fcm_dpo/q_t": 0.49563997983932495, + "grad_norm": 78.14775848388672, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.4897007942199707, + "logits/rejected": -0.45108163356781006, + "logps/chosen": -67.63417053222656, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.4769287109375, + "loss": 1.3691, + "margin_dpo/margin_mean": 0.17442861199378967, + "margin_dpo/margin_std": 0.2631375789642334, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.00836181640625, + "KL/mean": -0.017851993441581726, + "KL/rejected_KL_mean": -0.044071197509765625, + "KL/std": 0.251241534948349, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.052427202463150024, + "fcm_dpo/q_t": 0.49868953227996826, + "grad_norm": 81.75630950927734, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.4999546706676483, + "logits/rejected": -0.4452134966850281, + "logps/chosen": -47.73112869262695, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.516357421875, + "loss": 1.3813, + "margin_dpo/margin_mean": 0.05242687463760376, + "margin_dpo/margin_std": 0.3272395133972168, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.011135101318359375, + "KL/mean": -0.01634325087070465, + "KL/rejected_KL_mean": -0.043811798095703125, + "KL/std": 0.2484726905822754, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05494299530982971, + "fcm_dpo/q_t": 0.49862709641456604, + "grad_norm": 73.23149871826172, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.4927639365196228, + "logits/rejected": -0.44303786754608154, + "logps/chosen": -70.19422912597656, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.80140686035156, + "loss": 1.3812, + "margin_dpo/margin_mean": 0.05494317412376404, + "margin_dpo/margin_std": 0.3772922456264496, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.0436248779296875, + "KL/mean": -0.04940740764141083, + "KL/rejected_KL_mean": -0.055187225341796875, + "KL/std": 0.25541043281555176, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011567741632461548, + "fcm_dpo/q_t": 0.4997102618217468, + "grad_norm": 74.0939712524414, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.565744161605835, + "logits/rejected": -0.5108603835105896, + "logps/chosen": -50.846866607666016, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.8785400390625, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.0115680992603302, + "margin_dpo/margin_std": 0.34970372915267944, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.022800445556640625, + "KL/mean": -0.06389857083559036, + "KL/rejected_KL_mean": -0.1049957275390625, + "KL/std": 0.2660517394542694, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08219937980175018, + "fcm_dpo/q_t": 0.49794578552246094, + "grad_norm": 77.65446472167969, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.5047143697738647, + "logits/rejected": -0.481604665517807, + "logps/chosen": -50.0858154296875, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.97378540039062, + "loss": 1.3784, + "margin_dpo/margin_mean": 0.0821990817785263, + "margin_dpo/margin_std": 0.3332855701446533, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.06880569458007812, + "KL/mean": -0.04274939000606537, + "KL/rejected_KL_mean": -0.15430831909179688, + "KL/std": 0.31657668948173523, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22311243414878845, + "fcm_dpo/q_t": 0.4944257140159607, + "grad_norm": 83.68839263916016, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.49494728446006775, + "logits/rejected": -0.45161426067352295, + "logps/chosen": -58.98883056640625, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.65898132324219, + "loss": 1.3647, + "margin_dpo/margin_mean": 0.2231123149394989, + "margin_dpo/margin_std": 0.46688035130500793, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.05562019348144531, + "KL/mean": -0.0632062703371048, + "KL/rejected_KL_mean": -0.18203353881835938, + "KL/std": 0.3174479305744171, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2376532256603241, + "fcm_dpo/q_t": 0.4940638542175293, + "grad_norm": 80.2894515991211, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5025454759597778, + "logits/rejected": -0.480760395526886, + "logps/chosen": -60.02207565307617, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.32158660888672, + "loss": 1.3631, + "margin_dpo/margin_mean": 0.23765289783477783, + "margin_dpo/margin_std": 0.42710942029953003, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.021366119384765625, + "KL/mean": -0.09527391940355301, + "KL/rejected_KL_mean": -0.21192169189453125, + "KL/std": 0.2744912803173065, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23328350484371185, + "fcm_dpo/q_t": 0.4941703677177429, + "grad_norm": 81.72077178955078, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5121709108352661, + "logits/rejected": -0.49551209807395935, + "logps/chosen": -44.26966857910156, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.3371353149414, + "loss": 1.3635, + "margin_dpo/margin_mean": 0.2332831621170044, + "margin_dpo/margin_std": 0.3856799602508545, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.019725799560546875, + "KL/mean": -0.07422037422657013, + "KL/rejected_KL_mean": -0.16816329956054688, + "KL/std": 0.3426307439804077, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.18788549304008484, + "fcm_dpo/q_t": 0.49530547857284546, + "grad_norm": 74.0044937133789, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.5040639638900757, + "logits/rejected": -0.47433674335479736, + "logps/chosen": -52.51732635498047, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.5103530883789, + "loss": 1.368, + "margin_dpo/margin_mean": 0.18788480758666992, + "margin_dpo/margin_std": 0.4178956151008606, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.031414031982421875, + "KL/mean": -0.13168781995773315, + "KL/rejected_KL_mean": -0.29479217529296875, + "KL/std": 0.4489726126194, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3262089490890503, + "fcm_dpo/q_t": 0.49185606837272644, + "grad_norm": 87.91612243652344, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5216891765594482, + "logits/rejected": -0.48978933691978455, + "logps/chosen": -53.891395568847656, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.65451049804688, + "loss": 1.3548, + "margin_dpo/margin_mean": 0.3262087106704712, + "margin_dpo/margin_std": 0.5640857219696045, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.12083816528320312, + "KL/mean": -0.12519976496696472, + "KL/rejected_KL_mean": -0.37123870849609375, + "KL/std": 0.46627822518348694, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4920726418495178, + "fcm_dpo/q_t": 0.48771440982818604, + "grad_norm": 93.49578094482422, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5265411138534546, + "logits/rejected": -0.4901958703994751, + "logps/chosen": -42.77769088745117, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.09544372558594, + "loss": 1.3386, + "margin_dpo/margin_mean": 0.49207237362861633, + "margin_dpo/margin_std": 0.5811110734939575, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.023447036743164062, + "KL/mean": -0.1349368840456009, + "KL/rejected_KL_mean": -0.2933158874511719, + "KL/std": 0.42183345556259155, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.31675973534584045, + "fcm_dpo/q_t": 0.4920859932899475, + "grad_norm": 75.2684555053711, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5211254954338074, + "logits/rejected": -0.46746256947517395, + "logps/chosen": -60.53305435180664, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.69442749023438, + "loss": 1.3559, + "margin_dpo/margin_mean": 0.31675976514816284, + "margin_dpo/margin_std": 0.6104538440704346, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.07365036010742188, + "KL/mean": -0.17818748950958252, + "KL/rejected_KL_mean": -0.4300270080566406, + "KL/std": 0.47136425971984863, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5036798119544983, + "fcm_dpo/q_t": 0.48742347955703735, + "grad_norm": 90.97443389892578, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5397048592567444, + "logits/rejected": -0.49236124753952026, + "logps/chosen": -57.734134674072266, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.82437896728516, + "loss": 1.3374, + "margin_dpo/margin_mean": 0.5036802291870117, + "margin_dpo/margin_std": 0.5732718110084534, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.13624000549316406, + "KL/mean": -0.20667259395122528, + "KL/rejected_KL_mean": -0.549591064453125, + "KL/std": 0.6064626574516296, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.685826301574707, + "fcm_dpo/q_t": 0.48288410902023315, + "grad_norm": 87.44654846191406, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.49445396661758423, + "logits/rejected": -0.4637051820755005, + "logps/chosen": -52.441131591796875, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.0387954711914, + "loss": 1.32, + "margin_dpo/margin_mean": 0.6858261227607727, + "margin_dpo/margin_std": 0.6452517509460449, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.08217620849609375, + "KL/mean": -0.13851052522659302, + "KL/rejected_KL_mean": -0.35919952392578125, + "KL/std": 0.5338647365570068, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4413755536079407, + "fcm_dpo/q_t": 0.4889836013317108, + "grad_norm": 68.57612609863281, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.4767436385154724, + "logits/rejected": -0.4286072552204132, + "logps/chosen": -63.72474670410156, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.25320434570312, + "loss": 1.3437, + "margin_dpo/margin_mean": 0.44137513637542725, + "margin_dpo/margin_std": 0.639270544052124, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.18958282470703125, + "KL/mean": -0.1984645575284958, + "KL/rejected_KL_mean": -0.5865135192871094, + "KL/std": 0.7354353666305542, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7760944366455078, + "fcm_dpo/q_t": 0.4806801676750183, + "grad_norm": 82.93385314941406, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5040623545646667, + "logits/rejected": -0.4622010588645935, + "logps/chosen": -62.54994201660156, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.90401458740234, + "loss": 1.3124, + "margin_dpo/margin_mean": 0.7760943174362183, + "margin_dpo/margin_std": 0.9313377141952515, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.10210418701171875, + "KL/mean": -0.19319280982017517, + "KL/rejected_KL_mean": -0.48848724365234375, + "KL/std": 0.5709853172302246, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5905937552452087, + "fcm_dpo/q_t": 0.48525843024253845, + "grad_norm": 72.65520477294922, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.4709397256374359, + "logits/rejected": -0.44431936740875244, + "logps/chosen": -53.15886688232422, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.37362670898438, + "loss": 1.3292, + "margin_dpo/margin_mean": 0.5905939340591431, + "margin_dpo/margin_std": 0.6725684404373169, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.06002616882324219, + "KL/mean": -0.32852113246917725, + "KL/rejected_KL_mean": -0.717071533203125, + "KL/std": 0.7416244745254517, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7770934104919434, + "fcm_dpo/q_t": 0.4806344509124756, + "grad_norm": 78.09233093261719, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.4921151399612427, + "logits/rejected": -0.4746784567832947, + "logps/chosen": -50.757301330566406, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.638916015625, + "loss": 1.3121, + "margin_dpo/margin_mean": 0.7770930528640747, + "margin_dpo/margin_std": 0.8949083089828491, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.12356185913085938, + "KL/mean": -0.4787988066673279, + "KL/rejected_KL_mean": -1.0811653137207031, + "KL/std": 0.9876870512962341, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2047250270843506, + "fcm_dpo/q_t": 0.4700542688369751, + "grad_norm": 83.21550750732422, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5105268359184265, + "logits/rejected": -0.47380155324935913, + "logps/chosen": -50.90092849731445, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.90559387207031, + "loss": 1.2723, + "margin_dpo/margin_mean": 1.204724907875061, + "margin_dpo/margin_std": 1.0496253967285156, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.001544952392578125, + "KL/mean": -0.5374133586883545, + "KL/rejected_KL_mean": -1.0732803344726562, + "KL/std": 1.0570510625839233, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0717397928237915, + "fcm_dpo/q_t": 0.4733422100543976, + "grad_norm": 73.1773452758789, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5616889595985413, + "logits/rejected": -0.525577187538147, + "logps/chosen": -51.99303436279297, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.11390686035156, + "loss": 1.2857, + "margin_dpo/margin_mean": 1.0717395544052124, + "margin_dpo/margin_std": 1.1867928504943848, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.03598976135253906, + "KL/mean": -0.4841371178627014, + "KL/rejected_KL_mean": -1.0042648315429688, + "KL/std": 1.0527251958847046, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0402566194534302, + "fcm_dpo/q_t": 0.47418516874313354, + "grad_norm": 61.90176773071289, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5213212370872498, + "logits/rejected": -0.47842004895210266, + "logps/chosen": -62.7711181640625, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.89933776855469, + "loss": 1.2897, + "margin_dpo/margin_mean": 1.0402569770812988, + "margin_dpo/margin_std": 1.3224931955337524, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1625843048095703, + "KL/mean": -0.5425529479980469, + "KL/rejected_KL_mean": -1.2476882934570312, + "KL/std": 1.3779053688049316, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4102709293365479, + "fcm_dpo/q_t": 0.46522241830825806, + "grad_norm": 69.18618774414062, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5070066452026367, + "logits/rejected": -0.4741925597190857, + "logps/chosen": -48.227935791015625, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.16012573242188, + "loss": 1.2573, + "margin_dpo/margin_mean": 1.4102712869644165, + "margin_dpo/margin_std": 1.6879444122314453, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.11082649230957031, + "KL/mean": -0.7265699505805969, + "KL/rejected_KL_mean": -1.56396484375, + "KL/std": 1.2859086990356445, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6747896671295166, + "fcm_dpo/q_t": 0.4584406316280365, + "grad_norm": 74.18789672851562, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5621109008789062, + "logits/rejected": -0.5225807428359985, + "logps/chosen": -50.6396484375, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.13347625732422, + "loss": 1.2305, + "margin_dpo/margin_mean": 1.6747890710830688, + "margin_dpo/margin_std": 1.3528721332550049, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.1979236602783203, + "KL/mean": -0.589297354221344, + "KL/rejected_KL_mean": -1.3765220642089844, + "KL/std": 1.4053642749786377, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.574448585510254, + "fcm_dpo/q_t": 0.46115612983703613, + "grad_norm": 59.63584518432617, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.508709192276001, + "logits/rejected": -0.4779585897922516, + "logps/chosen": -57.78714370727539, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.67658996582031, + "loss": 1.2427, + "margin_dpo/margin_mean": 1.574448823928833, + "margin_dpo/margin_std": 1.7161786556243896, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.054210662841796875, + "KL/mean": -0.9238835573196411, + "KL/rejected_KL_mean": -1.9019737243652344, + "KL/std": 1.8625211715698242, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10009606182575226, + "fcm_dpo/delta": 0.09596265107393265, + "fcm_dpo/margin": 1.9561913013458252, + "fcm_dpo/q_t": 0.4519006013870239, + "grad_norm": 67.54621887207031, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.555192232131958, + "logits/rejected": -0.519508957862854, + "logps/chosen": -62.641605377197266, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.92550659179688, + "loss": 1.2101, + "margin_dpo/margin_mean": 1.956192135810852, + "margin_dpo/margin_std": 1.9915827512741089, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.2025127410888672, + "KL/mean": -1.0535614490509033, + "KL/rejected_KL_mean": -2.3096351623535156, + "KL/std": 2.123450517654419, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.10042069852352142, + "fcm_dpo/delta": 0.14809294044971466, + "fcm_dpo/margin": 2.512145519256592, + "fcm_dpo/q_t": 0.4385872483253479, + "grad_norm": 79.79023742675781, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5215531587600708, + "logits/rejected": -0.47460901737213135, + "logps/chosen": -58.763916015625, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.21800994873047, + "loss": 1.1643, + "margin_dpo/margin_mean": 2.51214599609375, + "margin_dpo/margin_std": 2.4093494415283203, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5451717376708984, + "KL/mean": -0.6989315748214722, + "KL/rejected_KL_mean": -1.9430427551269531, + "KL/std": 1.842559814453125, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.10070296376943588, + "fcm_dpo/delta": 0.14982065558433533, + "fcm_dpo/margin": 2.4882097244262695, + "fcm_dpo/q_t": 0.43854182958602905, + "grad_norm": 72.24876403808594, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5297493934631348, + "logits/rejected": -0.5043606162071228, + "logps/chosen": -53.61082458496094, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.42323303222656, + "loss": 1.1613, + "margin_dpo/margin_mean": 2.4882102012634277, + "margin_dpo/margin_std": 1.9865665435791016, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.26450538635253906, + "KL/mean": -1.1613378524780273, + "KL/rejected_KL_mean": -2.5871810913085938, + "KL/std": 2.3007254600524902, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.10095521807670593, + "fcm_dpo/delta": 0.11243705451488495, + "fcm_dpo/margin": 2.8516836166381836, + "fcm_dpo/q_t": 0.4297078847885132, + "grad_norm": 79.81075286865234, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.47335895895957947, + "logits/rejected": -0.45338839292526245, + "logps/chosen": -49.813995361328125, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.37094116210938, + "loss": 1.132, + "margin_dpo/margin_mean": 2.8516838550567627, + "margin_dpo/margin_std": 2.2934036254882812, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.16114044189453125, + "KL/mean": -1.0017555952072144, + "KL/rejected_KL_mean": -2.1646575927734375, + "KL/std": 2.1023666858673096, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.10128161311149597, + "fcm_dpo/delta": 0.16480150818824768, + "fcm_dpo/margin": 2.325796365737915, + "fcm_dpo/q_t": 0.4429783225059509, + "grad_norm": 62.990440368652344, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4691409766674042, + "logits/rejected": -0.45630985498428345, + "logps/chosen": -48.253787994384766, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.10108947753906, + "loss": 1.1822, + "margin_dpo/margin_mean": 2.325796604156494, + "margin_dpo/margin_std": 2.6178860664367676, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.17998123168945312, + "KL/mean": -1.29139244556427, + "KL/rejected_KL_mean": -2.762767791748047, + "KL/std": 2.6259617805480957, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.10152305662631989, + "fcm_dpo/delta": 0.10152298212051392, + "fcm_dpo/margin": 2.942751407623291, + "fcm_dpo/q_t": 0.42841678857803345, + "grad_norm": 69.16443634033203, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.546703577041626, + "logits/rejected": -0.4964791536331177, + "logps/chosen": -55.81944274902344, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.41535949707031, + "loss": 1.1354, + "margin_dpo/margin_mean": 2.9427504539489746, + "margin_dpo/margin_std": 3.2248454093933105, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.4117012023925781, + "KL/mean": -1.0690584182739258, + "KL/rejected_KL_mean": -2.5498199462890625, + "KL/std": 2.6058173179626465, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10173772275447845, + "fcm_dpo/delta": 0.09895431995391846, + "fcm_dpo/margin": 2.9615182876586914, + "fcm_dpo/q_t": 0.42702746391296387, + "grad_norm": 66.22404479980469, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5754678249359131, + "logits/rejected": -0.5229586958885193, + "logps/chosen": -57.51437759399414, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.2290267944336, + "loss": 1.1267, + "margin_dpo/margin_mean": 2.961519479751587, + "margin_dpo/margin_std": 2.71427583694458, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.04601478576660156, + "KL/mean": -1.533321738243103, + "KL/rejected_KL_mean": -3.112659454345703, + "KL/std": 2.705569267272949, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.10184906423091888, + "fcm_dpo/delta": 0.07855109125375748, + "fcm_dpo/margin": 3.15867018699646, + "fcm_dpo/q_t": 0.4223400354385376, + "grad_norm": 74.41484832763672, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.6021075248718262, + "logits/rejected": -0.5446429252624512, + "logps/chosen": -57.142059326171875, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.12925720214844, + "loss": 1.1158, + "margin_dpo/margin_mean": 3.158670663833618, + "margin_dpo/margin_std": 2.9422152042388916, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.3413887023925781, + "KL/mean": -1.5863913297653198, + "KL/rejected_KL_mean": -3.5141639709472656, + "KL/std": 3.4326961040496826, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.10196104645729065, + "fcm_dpo/delta": 0.0068893469870090485, + "fcm_dpo/margin": 3.8555495738983154, + "fcm_dpo/q_t": 0.4077719449996948, + "grad_norm": 73.93376159667969, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5720950365066528, + "logits/rejected": -0.5162076950073242, + "logps/chosen": -61.34387969970703, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.2816390991211, + "loss": 1.07, + "margin_dpo/margin_mean": 3.8555500507354736, + "margin_dpo/margin_std": 3.943361282348633, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.17385101318359375, + "KL/mean": -2.2524447441101074, + "KL/rejected_KL_mean": -4.3310394287109375, + "KL/std": 3.521286964416504, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10188421607017517, + "fcm_dpo/delta": -0.0236571803689003, + "fcm_dpo/margin": 4.157182693481445, + "fcm_dpo/q_t": 0.40021008253097534, + "grad_norm": 63.3592529296875, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5669504404067993, + "logits/rejected": -0.5317648649215698, + "logps/chosen": -58.897987365722656, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.6891860961914, + "loss": 1.0465, + "margin_dpo/margin_mean": 4.157183647155762, + "margin_dpo/margin_std": 4.036492347717285, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2824287414550781, + "KL/mean": -2.323167562484741, + "KL/rejected_KL_mean": -4.363910675048828, + "KL/std": 4.202240467071533, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10181613266468048, + "fcm_dpo/delta": -0.015670161694288254, + "fcm_dpo/margin": 4.081480979919434, + "fcm_dpo/q_t": 0.40601593255996704, + "grad_norm": 52.48577117919922, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5439983606338501, + "logits/rejected": -0.5120658278465271, + "logps/chosen": -61.656097412109375, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.36591339111328, + "loss": 1.0771, + "margin_dpo/margin_mean": 4.081480503082275, + "margin_dpo/margin_std": 5.182563781738281, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.3355236053466797, + "KL/mean": -2.707639694213867, + "KL/rejected_KL_mean": -5.750801086425781, + "KL/std": 4.7208662033081055, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.1015276089310646, + "fcm_dpo/delta": -0.2192670702934265, + "fcm_dpo/margin": 6.086328506469727, + "fcm_dpo/q_t": 0.3594849705696106, + "grad_norm": 59.986122131347656, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5709120035171509, + "logits/rejected": -0.5165004730224609, + "logps/chosen": -52.0018310546875, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.72471618652344, + "loss": 0.92, + "margin_dpo/margin_mean": 6.086329460144043, + "margin_dpo/margin_std": 5.089400291442871, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.19043922424316406, + "KL/mean": -3.303816795349121, + "KL/rejected_KL_mean": -6.4171905517578125, + "KL/std": 5.37472677230835, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.10113561153411865, + "fcm_dpo/delta": -0.23114511370658875, + "fcm_dpo/margin": 6.226757526397705, + "fcm_dpo/q_t": 0.36491042375564575, + "grad_norm": 58.598384857177734, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6268244385719299, + "logits/rejected": -0.6055992841720581, + "logps/chosen": -53.505088806152344, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.20079040527344, + "loss": 0.9452, + "margin_dpo/margin_mean": 6.226758003234863, + "margin_dpo/margin_std": 6.150996685028076, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.40209197998046875, + "KL/mean": -3.0939383506774902, + "KL/rejected_KL_mean": -5.785778045654297, + "KL/std": 5.071398735046387, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.10073995590209961, + "fcm_dpo/delta": -0.14311030507087708, + "fcm_dpo/margin": 5.3836894035339355, + "fcm_dpo/q_t": 0.3765104413032532, + "grad_norm": 55.20548629760742, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.614770770072937, + "logits/rejected": -0.5621797442436218, + "logps/chosen": -51.09075164794922, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.50117492675781, + "loss": 0.9724, + "margin_dpo/margin_mean": 5.383688926696777, + "margin_dpo/margin_std": 5.038286209106445, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9343166351318359, + "KL/mean": -4.156381607055664, + "KL/rejected_KL_mean": -7.378440856933594, + "KL/std": 6.175149917602539, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.10031983256340027, + "fcm_dpo/delta": -0.24800823628902435, + "fcm_dpo/margin": 6.444125652313232, + "fcm_dpo/q_t": 0.36249181628227234, + "grad_norm": 54.052001953125, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6286410093307495, + "logits/rejected": -0.5645859837532043, + "logps/chosen": -63.54955291748047, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.3719482421875, + "loss": 0.9603, + "margin_dpo/margin_mean": 6.444125652313232, + "margin_dpo/margin_std": 7.517010688781738, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.7007598876953125, + "KL/mean": -3.7873353958129883, + "KL/rejected_KL_mean": -6.873912811279297, + "KL/std": 6.059223175048828, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.09984228014945984, + "fcm_dpo/delta": -0.21765443682670593, + "fcm_dpo/margin": 6.173158645629883, + "fcm_dpo/q_t": 0.36800849437713623, + "grad_norm": 52.2566032409668, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6236534714698792, + "logits/rejected": -0.5816433429718018, + "logps/chosen": -58.633487701416016, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.04835510253906, + "loss": 0.9774, + "margin_dpo/margin_mean": 6.173159599304199, + "margin_dpo/margin_std": 7.336258888244629, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7435035705566406, + "KL/mean": -4.224997043609619, + "KL/rejected_KL_mean": -7.706493377685547, + "KL/std": 5.995540618896484, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.09933115541934967, + "fcm_dpo/delta": -0.29357433319091797, + "fcm_dpo/margin": 6.962983131408691, + "fcm_dpo/q_t": 0.34794965386390686, + "grad_norm": 53.632572174072266, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5823974609375, + "logits/rejected": -0.5546000003814697, + "logps/chosen": -71.23878479003906, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.2719497680664, + "loss": 0.905, + "margin_dpo/margin_mean": 6.962983131408691, + "margin_dpo/margin_std": 6.610713005065918, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.9821567535400391, + "KL/mean": -4.872446060180664, + "KL/rejected_KL_mean": -8.762733459472656, + "KL/std": 6.860911846160889, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.09864138066768646, + "fcm_dpo/delta": -0.37018388509750366, + "fcm_dpo/margin": 7.780574321746826, + "fcm_dpo/q_t": 0.3408956229686737, + "grad_norm": 57.22975540161133, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.6046779155731201, + "logits/rejected": -0.529869556427002, + "logps/chosen": -63.11509704589844, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.3800277709961, + "loss": 0.8917, + "margin_dpo/margin_mean": 7.780574798583984, + "margin_dpo/margin_std": 8.15798282623291, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.5007972717285156, + "KL/mean": -5.713290214538574, + "KL/rejected_KL_mean": -9.92578125, + "KL/std": 7.4406280517578125, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.0976877361536026, + "fcm_dpo/delta": -0.42706969380378723, + "fcm_dpo/margin": 8.424983978271484, + "fcm_dpo/q_t": 0.3317883610725403, + "grad_norm": 54.87312316894531, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6182709336280823, + "logits/rejected": -0.5774876475334167, + "logps/chosen": -53.43332290649414, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.81098175048828, + "loss": 0.866, + "margin_dpo/margin_mean": 8.4249849319458, + "margin_dpo/margin_std": 8.559152603149414, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.6551666259765625, + "KL/mean": -6.006525039672852, + "KL/rejected_KL_mean": -9.35788345336914, + "KL/std": 6.733491897583008, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.09715721011161804, + "fcm_dpo/delta": -0.25299882888793945, + "fcm_dpo/margin": 6.702718734741211, + "fcm_dpo/q_t": 0.357054203748703, + "grad_norm": 64.63815307617188, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6242314577102661, + "logits/rejected": -0.565211296081543, + "logps/chosen": -63.597354888916016, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.75128173828125, + "loss": 0.9594, + "margin_dpo/margin_mean": 6.702718734741211, + "margin_dpo/margin_std": 7.64279842376709, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.4079227447509766, + "KL/mean": -5.5540666580200195, + "KL/rejected_KL_mean": -9.700214385986328, + "KL/std": 9.089892387390137, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.09644635021686554, + "fcm_dpo/delta": -0.4031994640827179, + "fcm_dpo/margin": 8.292289733886719, + "fcm_dpo/q_t": 0.3487206697463989, + "grad_norm": 52.76113510131836, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6336618661880493, + "logits/rejected": -0.6002038717269897, + "logps/chosen": -62.041446685791016, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.5527114868164, + "loss": 0.9373, + "margin_dpo/margin_mean": 8.292289733886719, + "margin_dpo/margin_std": 11.241060256958008, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.6653404235839844, + "KL/mean": -4.752683639526367, + "KL/rejected_KL_mean": -7.840023040771484, + "KL/std": 6.594521999359131, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.09606001526117325, + "fcm_dpo/delta": -0.19424520432949066, + "fcm_dpo/margin": 6.174679279327393, + "fcm_dpo/q_t": 0.3748202323913574, + "grad_norm": 54.3487663269043, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.6234630942344666, + "logits/rejected": -0.590254545211792, + "logps/chosen": -57.81611251831055, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.40621948242188, + "loss": 1.0076, + "margin_dpo/margin_mean": 6.174679756164551, + "margin_dpo/margin_std": 8.05898666381836, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.6249332427978516, + "KL/mean": -6.814114570617676, + "KL/rejected_KL_mean": -11.0032958984375, + "KL/std": 8.401129722595215, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.09527701139450073, + "fcm_dpo/delta": -0.4018556475639343, + "fcm_dpo/margin": 8.378368377685547, + "fcm_dpo/q_t": 0.3336021900177002, + "grad_norm": 54.587913513183594, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6237167119979858, + "logits/rejected": -0.5801436901092529, + "logps/chosen": -75.77232360839844, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -108.61335754394531, + "loss": 0.8873, + "margin_dpo/margin_mean": 8.378368377685547, + "margin_dpo/margin_std": 8.819418907165527, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.9983386993408203, + "KL/mean": -5.894951820373535, + "KL/rejected_KL_mean": -10.79156494140625, + "KL/std": 9.023216247558594, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.0942838191986084, + "fcm_dpo/delta": -0.5291889905929565, + "fcm_dpo/margin": 9.79322338104248, + "fcm_dpo/q_t": 0.32112938165664673, + "grad_norm": 51.08706283569336, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5377709865570068, + "logits/rejected": -0.5033805966377258, + "logps/chosen": -54.99694061279297, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.32176208496094, + "loss": 0.8647, + "margin_dpo/margin_mean": 9.793224334716797, + "margin_dpo/margin_std": 11.081071853637695, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.154062271118164, + "KL/mean": -8.106063842773438, + "KL/rejected_KL_mean": -13.058067321777344, + "KL/std": 10.143718719482422, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.09334555268287659, + "fcm_dpo/delta": -0.5298206210136414, + "fcm_dpo/margin": 9.904004096984863, + "fcm_dpo/q_t": 0.3176112174987793, + "grad_norm": 51.96211624145508, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6737109422683716, + "logits/rejected": -0.6606993675231934, + "logps/chosen": -67.99006652832031, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.00453186035156, + "loss": 0.8728, + "margin_dpo/margin_mean": 9.904003143310547, + "margin_dpo/margin_std": 11.054542541503906, + "step": 65 + }, + { + "KL/chosen_KL_mean": -2.9128494262695312, + "KL/mean": -7.400773048400879, + "KL/rejected_KL_mean": -11.888694763183594, + "KL/std": 9.077596664428711, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.09246563911437988, + "fcm_dpo/delta": -0.4341561198234558, + "fcm_dpo/margin": 8.975846290588379, + "fcm_dpo/q_t": 0.3391925096511841, + "grad_norm": 52.89352035522461, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6639128923416138, + "logits/rejected": -0.6323233842849731, + "logps/chosen": -54.35637664794922, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -87.52499389648438, + "loss": 0.9072, + "margin_dpo/margin_mean": 8.975846290588379, + "margin_dpo/margin_std": 10.965092658996582, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.51031494140625, + "KL/mean": -7.159172058105469, + "KL/rejected_KL_mean": -11.808032989501953, + "KL/std": 9.121521949768066, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.09176331758499146, + "fcm_dpo/delta": -0.4569275379180908, + "fcm_dpo/margin": 9.297718048095703, + "fcm_dpo/q_t": 0.3344728648662567, + "grad_norm": 65.78748321533203, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.5981483459472656, + "logits/rejected": -0.5554331541061401, + "logps/chosen": -61.85111999511719, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -84.59532165527344, + "loss": 0.889, + "margin_dpo/margin_mean": 9.297718048095703, + "margin_dpo/margin_std": 10.999261856079102, + "step": 67 + }, + { + "KL/chosen_KL_mean": -2.786439895629883, + "KL/mean": -7.167520523071289, + "KL/rejected_KL_mean": -11.548595428466797, + "KL/std": 8.065038681030273, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.09100232273340225, + "fcm_dpo/delta": -0.4004414677619934, + "fcm_dpo/margin": 8.762155532836914, + "fcm_dpo/q_t": 0.3352241516113281, + "grad_norm": 49.70207977294922, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6351357698440552, + "logits/rejected": -0.5752934217453003, + "logps/chosen": -67.99227142333984, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -88.75584411621094, + "loss": 0.8714, + "margin_dpo/margin_mean": 8.762155532836914, + "margin_dpo/margin_std": 8.951116561889648, + "step": 68 + }, + { + "KL/chosen_KL_mean": -3.232574462890625, + "KL/mean": -8.498923301696777, + "KL/rejected_KL_mean": -13.765274047851562, + "KL/std": 9.480640411376953, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.08998025953769684, + "fcm_dpo/delta": -0.5533735752105713, + "fcm_dpo/margin": 10.532695770263672, + "fcm_dpo/q_t": 0.31352198123931885, + "grad_norm": 49.80611801147461, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.592496395111084, + "logits/rejected": -0.5690979957580566, + "logps/chosen": -63.051815032958984, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -117.15413665771484, + "loss": 0.8066, + "margin_dpo/margin_mean": 10.532697677612305, + "margin_dpo/margin_std": 10.455503463745117, + "step": 69 + }, + { + "KL/chosen_KL_mean": -4.626218795776367, + "KL/mean": -10.211868286132812, + "KL/rejected_KL_mean": -15.79751968383789, + "KL/std": 11.214607238769531, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.08902642130851746, + "fcm_dpo/delta": -0.6002567410469055, + "fcm_dpo/margin": 11.171297073364258, + "fcm_dpo/q_t": 0.3123650848865509, + "grad_norm": 55.3319206237793, + "learning_rate": 5e-07, + "logits/chosen": -0.6337051391601562, + "logits/rejected": -0.599523663520813, + "logps/chosen": -66.5568618774414, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -106.85830688476562, + "loss": 0.8238, + "margin_dpo/margin_mean": 11.171297073364258, + "margin_dpo/margin_std": 11.89487075805664, + "step": 70 + }, + { + "KL/chosen_KL_mean": -4.083652496337891, + "KL/mean": -10.58677864074707, + "KL/rejected_KL_mean": -17.089908599853516, + "KL/std": 11.582392692565918, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.08770347386598587, + "fcm_dpo/delta": -0.7494148015975952, + "fcm_dpo/margin": 13.006255149841309, + "fcm_dpo/q_t": 0.2887082099914551, + "grad_norm": 46.91083908081055, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6513431072235107, + "logits/rejected": -0.6108582019805908, + "logps/chosen": -65.83399200439453, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -114.42652893066406, + "loss": 0.7413, + "margin_dpo/margin_mean": 13.006256103515625, + "margin_dpo/margin_std": 12.625312805175781, + "step": 71 + }, + { + "KL/chosen_KL_mean": -4.482515335083008, + "KL/mean": -11.293744087219238, + "KL/rejected_KL_mean": -18.10497283935547, + "KL/std": 12.26617431640625, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.08633983135223389, + "fcm_dpo/delta": -0.7857062220573425, + "fcm_dpo/margin": 13.622451782226562, + "fcm_dpo/q_t": 0.28626900911331177, + "grad_norm": 54.487281799316406, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6568814516067505, + "logits/rejected": -0.6214373111724854, + "logps/chosen": -70.53592681884766, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -113.39196014404297, + "loss": 0.7684, + "margin_dpo/margin_mean": 13.622451782226562, + "margin_dpo/margin_std": 14.119553565979004, + "step": 72 + }, + { + "KL/chosen_KL_mean": -6.115819931030273, + "KL/mean": -12.82156753540039, + "KL/rejected_KL_mean": -19.52731704711914, + "KL/std": 15.655166625976562, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.0850212574005127, + "fcm_dpo/delta": -0.7493008375167847, + "fcm_dpo/margin": 13.41149616241455, + "fcm_dpo/q_t": 0.3212772309780121, + "grad_norm": 64.65656280517578, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6754893660545349, + "logits/rejected": -0.6334271430969238, + "logps/chosen": -72.3720932006836, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -109.98345184326172, + "loss": 0.9549, + "margin_dpo/margin_mean": 13.411497116088867, + "margin_dpo/margin_std": 19.28784942626953, + "step": 73 + }, + { + "KL/chosen_KL_mean": -6.437097549438477, + "KL/mean": -13.52927017211914, + "KL/rejected_KL_mean": -20.621444702148438, + "KL/std": 16.866043090820312, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.08371196687221527, + "fcm_dpo/delta": -0.7972096800804138, + "fcm_dpo/margin": 14.184351921081543, + "fcm_dpo/q_t": 0.3048827648162842, + "grad_norm": 60.71415328979492, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.652562141418457, + "logits/rejected": -0.6413716077804565, + "logps/chosen": -59.86198043823242, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -116.5683822631836, + "loss": 0.8886, + "margin_dpo/margin_mean": 14.184350967407227, + "margin_dpo/margin_std": 20.715850830078125, + "step": 74 + }, + { + "KL/chosen_KL_mean": -6.040624618530273, + "KL/mean": -15.964771270751953, + "KL/rejected_KL_mean": -25.888912200927734, + "KL/std": 17.338008880615234, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.0819825753569603, + "fcm_dpo/delta": -1.2461938858032227, + "fcm_dpo/margin": 19.84829330444336, + "fcm_dpo/q_t": 0.24623186886310577, + "grad_norm": 44.353580474853516, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6583486795425415, + "logits/rejected": -0.648951530456543, + "logps/chosen": -57.902286529541016, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -137.1428985595703, + "loss": 0.6504, + "margin_dpo/margin_mean": 19.84829330444336, + "margin_dpo/margin_std": 19.28493881225586, + "step": 75 + }, + { + "KL/chosen_KL_mean": -7.323387145996094, + "KL/mean": -13.676560401916504, + "KL/rejected_KL_mean": -20.029727935791016, + "KL/std": 13.826997756958008, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.080598384141922, + "fcm_dpo/delta": -0.630521833896637, + "fcm_dpo/margin": 12.706350326538086, + "fcm_dpo/q_t": 0.3035961389541626, + "grad_norm": 54.8695068359375, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6847676038742065, + "logits/rejected": -0.6407864689826965, + "logps/chosen": -60.58942413330078, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -98.24635314941406, + "loss": 0.8575, + "margin_dpo/margin_mean": 12.706350326538086, + "margin_dpo/margin_std": 14.94691276550293, + "step": 76 + }, + { + "KL/chosen_KL_mean": -6.192342758178711, + "KL/mean": -16.226093292236328, + "KL/rejected_KL_mean": -26.259841918945312, + "KL/std": 18.648479461669922, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.07882138341665268, + "fcm_dpo/delta": -1.2022576332092285, + "fcm_dpo/margin": 20.06749725341797, + "fcm_dpo/q_t": 0.24437257647514343, + "grad_norm": 62.43853759765625, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.681199848651886, + "logits/rejected": -0.6406725645065308, + "logps/chosen": -64.28902435302734, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -120.03345489501953, + "loss": 0.7151, + "margin_dpo/margin_mean": 20.06749725341797, + "margin_dpo/margin_std": 20.15266227722168, + "step": 77 + }, + { + "KL/chosen_KL_mean": -5.943998336791992, + "KL/mean": -13.652148246765137, + "KL/rejected_KL_mean": -21.36029815673828, + "KL/std": 14.54329776763916, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.07754114270210266, + "fcm_dpo/delta": -0.8045774698257446, + "fcm_dpo/margin": 15.41629695892334, + "fcm_dpo/q_t": 0.2894890308380127, + "grad_norm": 54.08612060546875, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6298891305923462, + "logits/rejected": -0.6018366813659668, + "logps/chosen": -61.55778503417969, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -106.29466247558594, + "loss": 0.7829, + "margin_dpo/margin_mean": 15.416296005249023, + "margin_dpo/margin_std": 16.38904571533203, + "step": 78 + }, + { + "KL/chosen_KL_mean": -5.85932731628418, + "KL/mean": -14.31186294555664, + "KL/rejected_KL_mean": -22.764400482177734, + "KL/std": 16.61045265197754, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.07614296674728394, + "fcm_dpo/delta": -0.8988029956817627, + "fcm_dpo/margin": 16.905065536499023, + "fcm_dpo/q_t": 0.2988874912261963, + "grad_norm": 47.835975646972656, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6631839275360107, + "logits/rejected": -0.6327118277549744, + "logps/chosen": -61.309814453125, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -110.4119644165039, + "loss": 0.8213, + "margin_dpo/margin_mean": 16.90506362915039, + "margin_dpo/margin_std": 20.20482635498047, + "step": 79 + }, + { + "KL/chosen_KL_mean": -8.03249740600586, + "KL/mean": -15.53980827331543, + "KL/rejected_KL_mean": -23.04711151123047, + "KL/std": 16.643917083740234, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.07485540211200714, + "fcm_dpo/delta": -0.7338055968284607, + "fcm_dpo/margin": 15.014616012573242, + "fcm_dpo/q_t": 0.31782710552215576, + "grad_norm": 52.25157928466797, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7018550634384155, + "logits/rejected": -0.654313325881958, + "logps/chosen": -66.55178833007812, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -110.59461212158203, + "loss": 0.8856, + "margin_dpo/margin_mean": 15.014616012573242, + "margin_dpo/margin_std": 19.10322380065918, + "step": 80 + }, + { + "KL/chosen_KL_mean": -6.677787780761719, + "KL/mean": -16.69403648376465, + "KL/rejected_KL_mean": -26.710281372070312, + "KL/std": 19.303951263427734, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.07341309636831284, + "fcm_dpo/delta": -1.088017225265503, + "fcm_dpo/margin": 20.032493591308594, + "fcm_dpo/q_t": 0.2700031101703644, + "grad_norm": 57.71792221069336, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7033861875534058, + "logits/rejected": -0.6704167127609253, + "logps/chosen": -73.12665557861328, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -156.37298583984375, + "loss": 0.7476, + "margin_dpo/margin_mean": 20.032493591308594, + "margin_dpo/margin_std": 22.697399139404297, + "step": 81 + }, + { + "KL/chosen_KL_mean": -8.434524536132812, + "KL/mean": -16.51101303100586, + "KL/rejected_KL_mean": -24.587501525878906, + "KL/std": 17.480518341064453, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.07238469272851944, + "fcm_dpo/delta": -0.7775775194168091, + "fcm_dpo/margin": 16.152978897094727, + "fcm_dpo/q_t": 0.31019556522369385, + "grad_norm": 67.83242797851562, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6252365112304688, + "logits/rejected": -0.6088253259658813, + "logps/chosen": -60.666908264160156, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -115.33074951171875, + "loss": 0.9764, + "margin_dpo/margin_mean": 16.152978897094727, + "margin_dpo/margin_std": 23.59215545654297, + "step": 82 + }, + { + "KL/chosen_KL_mean": -8.333160400390625, + "KL/mean": -17.423595428466797, + "KL/rejected_KL_mean": -26.514034271240234, + "KL/std": 17.845550537109375, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.0709674134850502, + "fcm_dpo/delta": -0.9022894501686096, + "fcm_dpo/margin": 18.180870056152344, + "fcm_dpo/q_t": 0.28802990913391113, + "grad_norm": 56.788631439208984, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6738285422325134, + "logits/rejected": -0.6303175091743469, + "logps/chosen": -64.1605453491211, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -130.2299346923828, + "loss": 0.8012, + "margin_dpo/margin_mean": 18.180870056152344, + "margin_dpo/margin_std": 20.579116821289062, + "step": 83 + }, + { + "KL/chosen_KL_mean": -6.720209121704102, + "KL/mean": -14.866608619689941, + "KL/rejected_KL_mean": -23.013004302978516, + "KL/std": 15.157805442810059, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.06980033963918686, + "fcm_dpo/delta": -0.746986985206604, + "fcm_dpo/margin": 16.292800903320312, + "fcm_dpo/q_t": 0.2837449908256531, + "grad_norm": 45.288509368896484, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6641340851783752, + "logits/rejected": -0.6268236041069031, + "logps/chosen": -73.89637756347656, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -110.31160736083984, + "loss": 0.7627, + "margin_dpo/margin_mean": 16.292800903320312, + "margin_dpo/margin_std": 15.4896879196167, + "step": 84 + }, + { + "KL/chosen_KL_mean": -7.11982536315918, + "KL/mean": -15.16385555267334, + "KL/rejected_KL_mean": -23.207889556884766, + "KL/std": 15.246694564819336, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.06898517906665802, + "fcm_dpo/delta": -0.71718430519104, + "fcm_dpo/margin": 16.08806610107422, + "fcm_dpo/q_t": 0.30644503235816956, + "grad_norm": 49.98605728149414, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6249376535415649, + "logits/rejected": -0.5925195217132568, + "logps/chosen": -65.52644348144531, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -101.84669494628906, + "loss": 0.8304, + "margin_dpo/margin_mean": 16.088064193725586, + "margin_dpo/margin_std": 17.78365707397461, + "step": 85 + }, + { + "KL/chosen_KL_mean": -8.913679122924805, + "KL/mean": -17.20705795288086, + "KL/rejected_KL_mean": -25.50043487548828, + "KL/std": 18.97850799560547, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.06793263554573059, + "fcm_dpo/delta": -0.7346456050872803, + "fcm_dpo/margin": 16.58675765991211, + "fcm_dpo/q_t": 0.3287374973297119, + "grad_norm": 61.370567321777344, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6779404878616333, + "logits/rejected": -0.6616241931915283, + "logps/chosen": -65.0511474609375, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -113.62208557128906, + "loss": 0.9688, + "margin_dpo/margin_mean": 16.586761474609375, + "margin_dpo/margin_std": 25.24515151977539, + "step": 86 + }, + { + "KL/chosen_KL_mean": -8.652029037475586, + "KL/mean": -17.837005615234375, + "KL/rejected_KL_mean": -27.021984100341797, + "KL/std": 18.654273986816406, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.0667271539568901, + "fcm_dpo/delta": -0.8370223045349121, + "fcm_dpo/margin": 18.369953155517578, + "fcm_dpo/q_t": 0.291393518447876, + "grad_norm": 48.349266052246094, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7131370306015015, + "logits/rejected": -0.678674578666687, + "logps/chosen": -64.28812408447266, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -122.48956298828125, + "loss": 0.8458, + "margin_dpo/margin_mean": 18.369953155517578, + "margin_dpo/margin_std": 22.00775146484375, + "step": 87 + }, + { + "KL/chosen_KL_mean": -10.71649169921875, + "KL/mean": -17.68636703491211, + "KL/rejected_KL_mean": -24.656238555908203, + "KL/std": 18.418485641479492, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.06592415273189545, + "fcm_dpo/delta": -0.5245096683502197, + "fcm_dpo/margin": 13.939743995666504, + "fcm_dpo/q_t": 0.33586132526397705, + "grad_norm": 54.128517150878906, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6804838180541992, + "logits/rejected": -0.666261613368988, + "logps/chosen": -84.38764190673828, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -131.36474609375, + "loss": 0.9729, + "margin_dpo/margin_mean": 13.939742088317871, + "margin_dpo/margin_std": 18.755279541015625, + "step": 88 + }, + { + "KL/chosen_KL_mean": -5.845853805541992, + "KL/mean": -13.205821990966797, + "KL/rejected_KL_mean": -20.565792083740234, + "KL/std": 15.079856872558594, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.06531459838151932, + "fcm_dpo/delta": -0.5665473937988281, + "fcm_dpo/margin": 14.719932556152344, + "fcm_dpo/q_t": 0.31483134627342224, + "grad_norm": 41.89472198486328, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7330664992332458, + "logits/rejected": -0.6961890459060669, + "logps/chosen": -66.47076416015625, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -102.64933776855469, + "loss": 0.832, + "margin_dpo/margin_mean": 14.719932556152344, + "margin_dpo/margin_std": 16.25090217590332, + "step": 89 + }, + { + "KL/chosen_KL_mean": -8.07003402709961, + "KL/mean": -17.208324432373047, + "KL/rejected_KL_mean": -26.346614837646484, + "KL/std": 19.909420013427734, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.06440868973731995, + "fcm_dpo/delta": -0.7857111692428589, + "fcm_dpo/margin": 18.276578903198242, + "fcm_dpo/q_t": 0.3148949146270752, + "grad_norm": 48.3042106628418, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.649189829826355, + "logits/rejected": -0.6456696391105652, + "logps/chosen": -61.3553466796875, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -137.89132690429688, + "loss": 0.8919, + "margin_dpo/margin_mean": 18.276578903198242, + "margin_dpo/margin_std": 24.311614990234375, + "step": 90 + }, + { + "KL/chosen_KL_mean": -7.308561325073242, + "KL/mean": -16.21691131591797, + "KL/rejected_KL_mean": -25.125263214111328, + "KL/std": 17.274951934814453, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.06340880692005157, + "fcm_dpo/delta": -0.7376101613044739, + "fcm_dpo/margin": 17.81670379638672, + "fcm_dpo/q_t": 0.30771857500076294, + "grad_norm": 46.95905303955078, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.679593563079834, + "logits/rejected": -0.64009690284729, + "logps/chosen": -69.11151885986328, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -112.99922180175781, + "loss": 0.8445, + "margin_dpo/margin_mean": 17.81670379638672, + "margin_dpo/margin_std": 20.871875762939453, + "step": 91 + }, + { + "KL/chosen_KL_mean": -6.844266891479492, + "KL/mean": -15.2471284866333, + "KL/rejected_KL_mean": -23.64999008178711, + "KL/std": 17.01289939880371, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.06255876272916794, + "fcm_dpo/delta": -0.6578457355499268, + "fcm_dpo/margin": 16.805727005004883, + "fcm_dpo/q_t": 0.31694501638412476, + "grad_norm": 42.78397750854492, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.7058894634246826, + "logits/rejected": -0.6780867576599121, + "logps/chosen": -58.48503494262695, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -101.53117370605469, + "loss": 0.8737, + "margin_dpo/margin_mean": 16.805727005004883, + "margin_dpo/margin_std": 20.510658264160156, + "step": 92 + }, + { + "KL/chosen_KL_mean": -7.523628234863281, + "KL/mean": -16.00517463684082, + "KL/rejected_KL_mean": -24.48672103881836, + "KL/std": 16.39925765991211, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.06168012320995331, + "fcm_dpo/delta": -0.6532446146011353, + "fcm_dpo/margin": 16.963092803955078, + "fcm_dpo/q_t": 0.30424731969833374, + "grad_norm": 38.19730758666992, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7320306301116943, + "logits/rejected": -0.6725261211395264, + "logps/chosen": -60.0528678894043, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -101.64746856689453, + "loss": 0.7985, + "margin_dpo/margin_mean": 16.963092803955078, + "margin_dpo/margin_std": 18.316625595092773, + "step": 93 + }, + { + "KL/chosen_KL_mean": -8.212984085083008, + "KL/mean": -17.419069290161133, + "KL/rejected_KL_mean": -26.62515640258789, + "KL/std": 17.246620178222656, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.06077170744538307, + "fcm_dpo/delta": -0.727825403213501, + "fcm_dpo/margin": 18.41216468811035, + "fcm_dpo/q_t": 0.2924576997756958, + "grad_norm": 39.58716583251953, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6868919730186462, + "logits/rejected": -0.6612251400947571, + "logps/chosen": -69.43559265136719, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -126.22418975830078, + "loss": 0.7625, + "margin_dpo/margin_mean": 18.41216468811035, + "margin_dpo/margin_std": 17.809844970703125, + "step": 94 + }, + { + "KL/chosen_KL_mean": -6.819789886474609, + "KL/mean": -14.332259178161621, + "KL/rejected_KL_mean": -21.8447322845459, + "KL/std": 15.010696411132812, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.06012392416596413, + "fcm_dpo/delta": -0.5081071257591248, + "fcm_dpo/margin": 15.02493953704834, + "fcm_dpo/q_t": 0.3186376094818115, + "grad_norm": 39.772193908691406, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.7156262397766113, + "logits/rejected": -0.6801230907440186, + "logps/chosen": -59.34343719482422, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -97.72508239746094, + "loss": 0.8464, + "margin_dpo/margin_mean": 15.024938583374023, + "margin_dpo/margin_std": 15.685343742370605, + "step": 95 + }, + { + "KL/chosen_KL_mean": -6.052604675292969, + "KL/mean": -16.015180587768555, + "KL/rejected_KL_mean": -25.977760314941406, + "KL/std": 18.94532012939453, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.05923323705792427, + "fcm_dpo/delta": -0.7903412580490112, + "fcm_dpo/margin": 19.925155639648438, + "fcm_dpo/q_t": 0.285098135471344, + "grad_norm": 38.21025085449219, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7016664743423462, + "logits/rejected": -0.6627290844917297, + "logps/chosen": -68.20957946777344, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -122.57377624511719, + "loss": 0.7422, + "margin_dpo/margin_mean": 19.925155639648438, + "margin_dpo/margin_std": 20.316211700439453, + "step": 96 + }, + { + "KL/chosen_KL_mean": -7.0335540771484375, + "KL/mean": -15.153478622436523, + "KL/rejected_KL_mean": -23.273406982421875, + "KL/std": 15.246045112609863, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.05863576382398605, + "fcm_dpo/delta": -0.5571595430374146, + "fcm_dpo/margin": 16.239856719970703, + "fcm_dpo/q_t": 0.30908551812171936, + "grad_norm": 40.11430358886719, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6307961940765381, + "logits/rejected": -0.5752667188644409, + "logps/chosen": -61.6799201965332, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -100.2381591796875, + "loss": 0.809, + "margin_dpo/margin_mean": 16.239856719970703, + "margin_dpo/margin_std": 15.091365814208984, + "step": 97 + }, + { + "KL/chosen_KL_mean": -8.594707489013672, + "KL/mean": -17.450077056884766, + "KL/rejected_KL_mean": -26.305450439453125, + "KL/std": 16.9114933013916, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.05783893167972565, + "fcm_dpo/delta": -0.6308857202529907, + "fcm_dpo/margin": 17.710742950439453, + "fcm_dpo/q_t": 0.3036477863788605, + "grad_norm": 41.21179962158203, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.7042789459228516, + "logits/rejected": -0.6825248003005981, + "logps/chosen": -73.85333251953125, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -122.83293914794922, + "loss": 0.8434, + "margin_dpo/margin_mean": 17.710742950439453, + "margin_dpo/margin_std": 19.316356658935547, + "step": 98 + }, + { + "KL/chosen_KL_mean": -7.319267272949219, + "KL/mean": -17.318824768066406, + "KL/rejected_KL_mean": -27.318382263183594, + "KL/std": 18.239498138427734, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.05701300501823425, + "fcm_dpo/delta": -0.7488125562667847, + "fcm_dpo/margin": 19.999114990234375, + "fcm_dpo/q_t": 0.29604774713516235, + "grad_norm": 37.882423400878906, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.677663266658783, + "logits/rejected": -0.6690654754638672, + "logps/chosen": -52.9577522277832, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -113.75631713867188, + "loss": 0.7645, + "margin_dpo/margin_mean": 19.999114990234375, + "margin_dpo/margin_std": 21.571836471557617, + "step": 99 + }, + { + "KL/chosen_KL_mean": -7.7525177001953125, + "KL/mean": -14.72110652923584, + "KL/rejected_KL_mean": -21.689693450927734, + "KL/std": 15.243199348449707, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.056496504694223404, + "fcm_dpo/delta": -0.3903456926345825, + "fcm_dpo/margin": 13.937179565429688, + "fcm_dpo/q_t": 0.3490157425403595, + "grad_norm": 44.29375076293945, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6663979291915894, + "logits/rejected": -0.6204314231872559, + "logps/chosen": -65.34649658203125, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -95.74990844726562, + "loss": 0.9793, + "margin_dpo/margin_mean": 13.937179565429688, + "margin_dpo/margin_std": 19.24066162109375, + "step": 100 + }, + { + "KL/chosen_KL_mean": -9.333072662353516, + "KL/mean": -17.382938385009766, + "KL/rejected_KL_mean": -25.43280029296875, + "KL/std": 17.255746841430664, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.05588849261403084, + "fcm_dpo/delta": -0.5049155950546265, + "fcm_dpo/margin": 16.0997257232666, + "fcm_dpo/q_t": 0.33638375997543335, + "grad_norm": 41.45513153076172, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6803932189941406, + "logits/rejected": -0.6409810185432434, + "logps/chosen": -70.98192596435547, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -108.62248229980469, + "loss": 0.9124, + "margin_dpo/margin_mean": 16.0997257232666, + "margin_dpo/margin_std": 20.765628814697266, + "step": 101 + }, + { + "KL/chosen_KL_mean": -11.021684646606445, + "KL/mean": -17.180133819580078, + "KL/rejected_KL_mean": -23.338584899902344, + "KL/std": 16.58172607421875, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.05561038851737976, + "fcm_dpo/delta": -0.286824107170105, + "fcm_dpo/margin": 12.316900253295898, + "fcm_dpo/q_t": 0.367995947599411, + "grad_norm": 49.00424575805664, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.7246113419532776, + "logits/rejected": -0.6774600744247437, + "logps/chosen": -75.1005630493164, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -91.52566528320312, + "loss": 1.0377, + "margin_dpo/margin_mean": 12.316900253295898, + "margin_dpo/margin_std": 18.861268997192383, + "step": 102 + }, + { + "KL/chosen_KL_mean": -6.950771331787109, + "KL/mean": -18.078916549682617, + "KL/rejected_KL_mean": -29.207061767578125, + "KL/std": 18.45110511779785, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.0547785758972168, + "fcm_dpo/delta": -0.8294910192489624, + "fcm_dpo/margin": 22.25628662109375, + "fcm_dpo/q_t": 0.27520689368247986, + "grad_norm": 35.01582336425781, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.7053004503250122, + "logits/rejected": -0.6689953804016113, + "logps/chosen": -68.25004577636719, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -122.77976989746094, + "loss": 0.7045, + "margin_dpo/margin_mean": 22.25628662109375, + "margin_dpo/margin_std": 20.460426330566406, + "step": 103 + }, + { + "KL/chosen_KL_mean": -8.538637161254883, + "KL/mean": -19.099098205566406, + "KL/rejected_KL_mean": -29.659557342529297, + "KL/std": 20.309539794921875, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.05398762971162796, + "fcm_dpo/delta": -0.748435378074646, + "fcm_dpo/margin": 21.12091827392578, + "fcm_dpo/q_t": 0.301607608795166, + "grad_norm": 38.46689224243164, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.7184367775917053, + "logits/rejected": -0.6757808923721313, + "logps/chosen": -62.91141128540039, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -119.22427368164062, + "loss": 0.8037, + "margin_dpo/margin_mean": 21.12091827392578, + "margin_dpo/margin_std": 23.370576858520508, + "step": 104 + }, + { + "KL/chosen_KL_mean": -7.210062026977539, + "KL/mean": -21.615440368652344, + "KL/rejected_KL_mean": -36.02082061767578, + "KL/std": 20.99075698852539, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.05285883694887161, + "fcm_dpo/delta": -1.1399750709533691, + "fcm_dpo/margin": 28.810760498046875, + "fcm_dpo/q_t": 0.22892296314239502, + "grad_norm": 32.862518310546875, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.6998032331466675, + "logits/rejected": -0.6635599136352539, + "logps/chosen": -61.84900665283203, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -133.9943389892578, + "loss": 0.5761, + "margin_dpo/margin_mean": 28.810760498046875, + "margin_dpo/margin_std": 22.290760040283203, + "step": 105 + }, + { + "KL/chosen_KL_mean": -8.725595474243164, + "KL/mean": -18.029333114624023, + "KL/rejected_KL_mean": -27.333072662353516, + "KL/std": 17.50006675720215, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.05209173262119293, + "fcm_dpo/delta": -0.5748573541641235, + "fcm_dpo/margin": 18.607471466064453, + "fcm_dpo/q_t": 0.31074488162994385, + "grad_norm": 37.280517578125, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.6548305749893188, + "logits/rejected": -0.6039139628410339, + "logps/chosen": -63.55849075317383, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -112.55769348144531, + "loss": 0.8189, + "margin_dpo/margin_mean": 18.607471466064453, + "margin_dpo/margin_std": 19.15203094482422, + "step": 106 + }, + { + "KL/chosen_KL_mean": -10.663707733154297, + "KL/mean": -21.222810745239258, + "KL/rejected_KL_mean": -31.781909942626953, + "KL/std": 20.50044822692871, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.05137866735458374, + "fcm_dpo/delta": -0.6929377913475037, + "fcm_dpo/margin": 21.118200302124023, + "fcm_dpo/q_t": 0.2957811951637268, + "grad_norm": 35.688934326171875, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.7083314657211304, + "logits/rejected": -0.6538152694702148, + "logps/chosen": -80.37152099609375, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -126.52142333984375, + "loss": 0.7917, + "margin_dpo/margin_mean": 21.118200302124023, + "margin_dpo/margin_std": 21.160049438476562, + "step": 107 + }, + { + "KL/chosen_KL_mean": -10.208215713500977, + "KL/mean": -21.411640167236328, + "KL/rejected_KL_mean": -32.61506652832031, + "KL/std": 21.554988861083984, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.05065499246120453, + "fcm_dpo/delta": -0.743513822555542, + "fcm_dpo/margin": 22.40685272216797, + "fcm_dpo/q_t": 0.31041109561920166, + "grad_norm": 40.97300338745117, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.7138278484344482, + "logits/rejected": -0.671709418296814, + "logps/chosen": -66.21810150146484, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -128.41107177734375, + "loss": 0.8552, + "margin_dpo/margin_mean": 22.40685272216797, + "margin_dpo/margin_std": 26.986148834228516, + "step": 108 + }, + { + "KL/chosen_KL_mean": -9.108814239501953, + "KL/mean": -21.646095275878906, + "KL/rejected_KL_mean": -34.183372497558594, + "KL/std": 21.880144119262695, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.04980740696191788, + "fcm_dpo/delta": -0.8599492907524109, + "fcm_dpo/margin": 25.074560165405273, + "fcm_dpo/q_t": 0.2714756727218628, + "grad_norm": 35.744224548339844, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.6477692127227783, + "logits/rejected": -0.6211438179016113, + "logps/chosen": -71.99430084228516, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -132.86911010742188, + "loss": 0.705, + "margin_dpo/margin_mean": 25.074560165405273, + "margin_dpo/margin_std": 22.3857421875, + "step": 109 + }, + { + "KL/chosen_KL_mean": -8.421268463134766, + "KL/mean": -19.950103759765625, + "KL/rejected_KL_mean": -31.478946685791016, + "KL/std": 21.274383544921875, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.048977963626384735, + "fcm_dpo/delta": -0.739526093006134, + "fcm_dpo/margin": 23.057676315307617, + "fcm_dpo/q_t": 0.29622316360473633, + "grad_norm": 35.101802825927734, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.6622586250305176, + "logits/rejected": -0.6132292151451111, + "logps/chosen": -67.17495727539062, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -111.22895812988281, + "loss": 0.811, + "margin_dpo/margin_mean": 23.05767822265625, + "margin_dpo/margin_std": 24.90774917602539, + "step": 110 + }, + { + "KL/chosen_KL_mean": -11.178991317749023, + "KL/mean": -23.28662109375, + "KL/rejected_KL_mean": -35.39425277709961, + "KL/std": 23.18701934814453, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.04827245697379112, + "fcm_dpo/delta": -0.7788447141647339, + "fcm_dpo/margin": 24.21526527404785, + "fcm_dpo/q_t": 0.2929428219795227, + "grad_norm": 37.42465591430664, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.690138578414917, + "logits/rejected": -0.6623334884643555, + "logps/chosen": -79.8031005859375, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -133.8231201171875, + "loss": 0.8051, + "margin_dpo/margin_mean": 24.21526336669922, + "margin_dpo/margin_std": 26.194927215576172, + "step": 111 + }, + { + "KL/chosen_KL_mean": -10.65353012084961, + "KL/mean": -19.160493850708008, + "KL/rejected_KL_mean": -27.66745376586914, + "KL/std": 17.888885498046875, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.04778105020523071, + "fcm_dpo/delta": -0.4168868958950043, + "fcm_dpo/margin": 17.01392364501953, + "fcm_dpo/q_t": 0.3416486978530884, + "grad_norm": 37.01518249511719, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.7056373357772827, + "logits/rejected": -0.6568832397460938, + "logps/chosen": -60.90317153930664, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -92.44187927246094, + "loss": 0.927, + "margin_dpo/margin_mean": 17.01392364501953, + "margin_dpo/margin_std": 20.706562042236328, + "step": 112 + }, + { + "KL/chosen_KL_mean": -11.571048736572266, + "KL/mean": -21.198585510253906, + "KL/rejected_KL_mean": -30.826129913330078, + "KL/std": 16.976173400878906, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.047352179884910583, + "fcm_dpo/delta": -0.5165420770645142, + "fcm_dpo/margin": 19.255077362060547, + "fcm_dpo/q_t": 0.31623178720474243, + "grad_norm": 35.05489730834961, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.6561819911003113, + "logits/rejected": -0.6042188405990601, + "logps/chosen": -78.28399658203125, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -108.79483032226562, + "loss": 0.8276, + "margin_dpo/margin_mean": 19.255077362060547, + "margin_dpo/margin_std": 18.763565063476562, + "step": 113 + }, + { + "KL/chosen_KL_mean": -10.532327651977539, + "KL/mean": -24.249805450439453, + "KL/rejected_KL_mean": -37.96728515625, + "KL/std": 25.60963249206543, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.04655773192644119, + "fcm_dpo/delta": -0.8899149894714355, + "fcm_dpo/margin": 27.43494987487793, + "fcm_dpo/q_t": 0.28191447257995605, + "grad_norm": 32.85068893432617, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.6922754049301147, + "logits/rejected": -0.6501392126083374, + "logps/chosen": -68.31741333007812, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -125.07695007324219, + "loss": 0.741, + "margin_dpo/margin_mean": 27.43494987487793, + "margin_dpo/margin_std": 29.690120697021484, + "step": 114 + }, + { + "KL/chosen_KL_mean": -14.748771667480469, + "KL/mean": -24.86666488647461, + "KL/rejected_KL_mean": -34.98455810546875, + "KL/std": 23.312055587768555, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.04605109244585037, + "fcm_dpo/delta": -0.5366820693016052, + "fcm_dpo/margin": 20.23578453063965, + "fcm_dpo/q_t": 0.3346002697944641, + "grad_norm": 47.72368621826172, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.7006338834762573, + "logits/rejected": -0.661476194858551, + "logps/chosen": -80.33141326904297, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -133.55007934570312, + "loss": 0.9509, + "margin_dpo/margin_mean": 20.23578643798828, + "margin_dpo/margin_std": 28.74742317199707, + "step": 115 + }, + { + "KL/chosen_KL_mean": -10.949459075927734, + "KL/mean": -21.22800064086914, + "KL/rejected_KL_mean": -31.506553649902344, + "KL/std": 19.726478576660156, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.04555881768465042, + "fcm_dpo/delta": -0.5414063930511475, + "fcm_dpo/margin": 20.55708885192871, + "fcm_dpo/q_t": 0.3143802285194397, + "grad_norm": 33.97036361694336, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.6851849555969238, + "logits/rejected": -0.6553069353103638, + "logps/chosen": -62.349769592285156, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -112.02840423583984, + "loss": 0.8086, + "margin_dpo/margin_mean": 20.55708885192871, + "margin_dpo/margin_std": 20.56753921508789, + "step": 116 + }, + { + "KL/chosen_KL_mean": -12.656818389892578, + "KL/mean": -22.359678268432617, + "KL/rejected_KL_mean": -32.06254196166992, + "KL/std": 20.812591552734375, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.045082706958055496, + "fcm_dpo/delta": -0.4791034758090973, + "fcm_dpo/margin": 19.405723571777344, + "fcm_dpo/q_t": 0.329871267080307, + "grad_norm": 42.32595443725586, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.6561278104782104, + "logits/rejected": -0.6137540340423584, + "logps/chosen": -81.95523071289062, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -98.64653015136719, + "loss": 0.8846, + "margin_dpo/margin_mean": 19.405723571777344, + "margin_dpo/margin_std": 22.938228607177734, + "step": 117 + }, + { + "KL/chosen_KL_mean": -11.995250701904297, + "KL/mean": -21.735275268554688, + "KL/rejected_KL_mean": -31.47528839111328, + "KL/std": 18.84764862060547, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.0446624681353569, + "fcm_dpo/delta": -0.4741324484348297, + "fcm_dpo/margin": 19.480043411254883, + "fcm_dpo/q_t": 0.32578057050704956, + "grad_norm": 34.60670471191406, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.7065751552581787, + "logits/rejected": -0.673068642616272, + "logps/chosen": -67.63623046875, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -107.14434814453125, + "loss": 0.849, + "margin_dpo/margin_mean": 19.480045318603516, + "margin_dpo/margin_std": 20.228229522705078, + "step": 118 + }, + { + "KL/chosen_KL_mean": -14.332466125488281, + "KL/mean": -24.414047241210938, + "KL/rejected_KL_mean": -34.49562072753906, + "KL/std": 20.509563446044922, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.044187530875205994, + "fcm_dpo/delta": -0.49593040347099304, + "fcm_dpo/margin": 20.16315460205078, + "fcm_dpo/q_t": 0.3306369185447693, + "grad_norm": 39.30467987060547, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.6826314330101013, + "logits/rejected": -0.659497857093811, + "logps/chosen": -87.84266662597656, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -137.47291564941406, + "loss": 0.8901, + "margin_dpo/margin_mean": 20.16315269470215, + "margin_dpo/margin_std": 24.12423324584961, + "step": 119 + }, + { + "KL/chosen_KL_mean": -15.493656158447266, + "KL/mean": -28.643577575683594, + "KL/rejected_KL_mean": -41.79349136352539, + "KL/std": 25.165164947509766, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.043614111840724945, + "fcm_dpo/delta": -0.7557896375656128, + "fcm_dpo/margin": 26.299835205078125, + "fcm_dpo/q_t": 0.28874871134757996, + "grad_norm": 35.36503219604492, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.7092448472976685, + "logits/rejected": -0.6688870787620544, + "logps/chosen": -92.27449035644531, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -149.81723022460938, + "loss": 0.7615, + "margin_dpo/margin_mean": 26.299835205078125, + "margin_dpo/margin_std": 26.114606857299805, + "step": 120 + }, + { + "KL/chosen_KL_mean": -14.7747802734375, + "KL/mean": -29.033899307250977, + "KL/rejected_KL_mean": -43.29301071166992, + "KL/std": 24.84239959716797, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.04289271682500839, + "fcm_dpo/delta": -0.8338814973831177, + "fcm_dpo/margin": 28.518238067626953, + "fcm_dpo/q_t": 0.2877199053764343, + "grad_norm": 35.2734260559082, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.6755824089050293, + "logits/rejected": -0.6621620655059814, + "logps/chosen": -76.5646743774414, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -153.28758239746094, + "loss": 0.7614, + "margin_dpo/margin_mean": 28.518238067626953, + "margin_dpo/margin_std": 29.589004516601562, + "step": 121 + }, + { + "KL/chosen_KL_mean": -13.738988876342773, + "KL/mean": -31.479476928710938, + "KL/rejected_KL_mean": -49.219966888427734, + "KL/std": 29.412010192871094, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.042031895369291306, + "fcm_dpo/delta": -1.1078605651855469, + "fcm_dpo/margin": 35.48097229003906, + "fcm_dpo/q_t": 0.23624783754348755, + "grad_norm": 34.3507194519043, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.6865692734718323, + "logits/rejected": -0.6875172257423401, + "logps/chosen": -60.641197204589844, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -155.9341583251953, + "loss": 0.601, + "margin_dpo/margin_mean": 35.48097229003906, + "margin_dpo/margin_std": 28.255516052246094, + "step": 122 + }, + { + "KL/chosen_KL_mean": -14.509866714477539, + "KL/mean": -26.49087905883789, + "KL/rejected_KL_mean": -38.47188949584961, + "KL/std": 25.016231536865234, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.04140360653400421, + "fcm_dpo/delta": -0.5984065532684326, + "fcm_dpo/margin": 23.962020874023438, + "fcm_dpo/q_t": 0.3241133689880371, + "grad_norm": 40.017181396484375, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.6745227575302124, + "logits/rejected": -0.645736038684845, + "logps/chosen": -75.8485107421875, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -126.24728393554688, + "loss": 0.9233, + "margin_dpo/margin_mean": 23.962020874023438, + "margin_dpo/margin_std": 31.976825714111328, + "step": 123 + }, + { + "KL/chosen_KL_mean": -15.476913452148438, + "KL/mean": -31.55095672607422, + "KL/rejected_KL_mean": -47.62500762939453, + "KL/std": 31.990467071533203, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.040722060948610306, + "fcm_dpo/delta": -0.9215083718299866, + "fcm_dpo/margin": 32.1480827331543, + "fcm_dpo/q_t": 0.29319748282432556, + "grad_norm": 40.438446044921875, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.6482193470001221, + "logits/rejected": -0.6424369215965271, + "logps/chosen": -86.92524719238281, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -165.20556640625, + "loss": 0.8082, + "margin_dpo/margin_mean": 32.1480827331543, + "margin_dpo/margin_std": 39.39332580566406, + "step": 124 + }, + { + "KL/chosen_KL_mean": -12.162187576293945, + "KL/mean": -27.15265655517578, + "KL/rejected_KL_mean": -42.14312744140625, + "KL/std": 26.1416072845459, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.04007040709257126, + "fcm_dpo/delta": -0.8108488917350769, + "fcm_dpo/margin": 29.980934143066406, + "fcm_dpo/q_t": 0.283088743686676, + "grad_norm": 34.17176055908203, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.6478005647659302, + "logits/rejected": -0.6321696043014526, + "logps/chosen": -62.29912567138672, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -126.13174438476562, + "loss": 0.7329, + "margin_dpo/margin_mean": 29.980934143066406, + "margin_dpo/margin_std": 28.95359992980957, + "step": 125 + }, + { + "KL/chosen_KL_mean": -13.959470748901367, + "KL/mean": -28.002531051635742, + "KL/rejected_KL_mean": -42.04559326171875, + "KL/std": 24.500999450683594, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.039464544504880905, + "fcm_dpo/delta": -0.7164196968078613, + "fcm_dpo/margin": 28.08612060546875, + "fcm_dpo/q_t": 0.2936287522315979, + "grad_norm": 35.180824279785156, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.6252586245536804, + "logits/rejected": -0.5968210101127625, + "logps/chosen": -69.62654113769531, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -140.17535400390625, + "loss": 0.7744, + "margin_dpo/margin_mean": 28.086122512817383, + "margin_dpo/margin_std": 28.475379943847656, + "step": 126 + }, + { + "KL/chosen_KL_mean": -12.439422607421875, + "KL/mean": -24.02037811279297, + "KL/rejected_KL_mean": -35.60133361816406, + "KL/std": 20.718502044677734, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.039042189717292786, + "fcm_dpo/delta": -0.508735179901123, + "fcm_dpo/margin": 23.161911010742188, + "fcm_dpo/q_t": 0.3189850449562073, + "grad_norm": 33.86922836303711, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.639170229434967, + "logits/rejected": -0.6085081100463867, + "logps/chosen": -68.99410247802734, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -112.39710998535156, + "loss": 0.8444, + "margin_dpo/margin_mean": 23.161911010742188, + "margin_dpo/margin_std": 23.607559204101562, + "step": 127 + }, + { + "KL/chosen_KL_mean": -13.538829803466797, + "KL/mean": -25.821094512939453, + "KL/rejected_KL_mean": -38.103355407714844, + "KL/std": 26.004104614257812, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.03862866759300232, + "fcm_dpo/delta": -0.5539049506187439, + "fcm_dpo/margin": 24.564537048339844, + "fcm_dpo/q_t": 0.3256795406341553, + "grad_norm": 36.7492790222168, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.6575911641120911, + "logits/rejected": -0.6155836582183838, + "logps/chosen": -71.6597900390625, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -114.54232025146484, + "loss": 0.8855, + "margin_dpo/margin_mean": 24.564537048339844, + "margin_dpo/margin_std": 29.753498077392578, + "step": 128 + }, + { + "KL/chosen_KL_mean": -18.56671905517578, + "KL/mean": -31.042112350463867, + "KL/rejected_KL_mean": -43.51750183105469, + "KL/std": 26.483261108398438, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.03819035738706589, + "fcm_dpo/delta": -0.5580515265464783, + "fcm_dpo/margin": 24.950782775878906, + "fcm_dpo/q_t": 0.3306121826171875, + "grad_norm": 41.7448616027832, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.6658838391304016, + "logits/rejected": -0.6404169201850891, + "logps/chosen": -85.48309326171875, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -140.1597137451172, + "loss": 0.9331, + "margin_dpo/margin_mean": 24.950782775878906, + "margin_dpo/margin_std": 33.41223907470703, + "step": 129 + }, + { + "KL/chosen_KL_mean": -13.968193054199219, + "KL/mean": -31.330284118652344, + "KL/rejected_KL_mean": -48.69237518310547, + "KL/std": 29.97433090209961, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.037624701857566833, + "fcm_dpo/delta": -0.9174715280532837, + "fcm_dpo/margin": 34.724178314208984, + "fcm_dpo/q_t": 0.27249062061309814, + "grad_norm": 30.657638549804688, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.6312265396118164, + "logits/rejected": -0.6224026083946228, + "logps/chosen": -58.63504409790039, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -131.47402954101562, + "loss": 0.7489, + "margin_dpo/margin_mean": 34.724178314208984, + "margin_dpo/margin_std": 33.30211639404297, + "step": 130 + }, + { + "KL/chosen_KL_mean": -10.705987930297852, + "KL/mean": -26.785293579101562, + "KL/rejected_KL_mean": -42.86460494995117, + "KL/std": 26.98633575439453, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.036931660026311874, + "fcm_dpo/delta": -0.7972490787506104, + "fcm_dpo/margin": 32.158607482910156, + "fcm_dpo/q_t": 0.2758054733276367, + "grad_norm": 30.52397346496582, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.6491000652313232, + "logits/rejected": -0.6438955068588257, + "logps/chosen": -55.630577087402344, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -131.30862426757812, + "loss": 0.7272, + "margin_dpo/margin_mean": 32.158607482910156, + "margin_dpo/margin_std": 29.393150329589844, + "step": 131 + }, + { + "KL/chosen_KL_mean": -15.432332992553711, + "KL/mean": -30.285429000854492, + "KL/rejected_KL_mean": -45.13852310180664, + "KL/std": 27.885330200195312, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.036396466195583344, + "fcm_dpo/delta": -0.6889567375183105, + "fcm_dpo/margin": 29.70619010925293, + "fcm_dpo/q_t": 0.30243855714797974, + "grad_norm": 34.44231414794922, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.667124330997467, + "logits/rejected": -0.6470081806182861, + "logps/chosen": -74.43341827392578, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -133.03067016601562, + "loss": 0.7983, + "margin_dpo/margin_mean": 29.706192016601562, + "margin_dpo/margin_std": 32.57990646362305, + "step": 132 + }, + { + "KL/chosen_KL_mean": -18.101097106933594, + "KL/mean": -31.565828323364258, + "KL/rejected_KL_mean": -45.03055191040039, + "KL/std": 27.50137710571289, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.03596626594662666, + "fcm_dpo/delta": -0.5742547512054443, + "fcm_dpo/margin": 26.929454803466797, + "fcm_dpo/q_t": 0.3249335289001465, + "grad_norm": 32.94694519042969, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.6543374061584473, + "logits/rejected": -0.6349329352378845, + "logps/chosen": -84.70559692382812, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -141.36410522460938, + "loss": 0.876, + "margin_dpo/margin_mean": 26.929454803466797, + "margin_dpo/margin_std": 33.04655456542969, + "step": 133 + }, + { + "KL/chosen_KL_mean": -13.69911003112793, + "KL/mean": -28.43798065185547, + "KL/rejected_KL_mean": -43.176849365234375, + "KL/std": 26.85441780090332, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.0355355329811573, + "fcm_dpo/delta": -0.654132604598999, + "fcm_dpo/margin": 29.477741241455078, + "fcm_dpo/q_t": 0.29827502369880676, + "grad_norm": 32.676300048828125, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.6648005247116089, + "logits/rejected": -0.6332652568817139, + "logps/chosen": -65.76836395263672, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -130.83135986328125, + "loss": 0.7592, + "margin_dpo/margin_mean": 29.477741241455078, + "margin_dpo/margin_std": 27.80714225769043, + "step": 134 + }, + { + "KL/chosen_KL_mean": -17.467365264892578, + "KL/mean": -34.69174575805664, + "KL/rejected_KL_mean": -51.9161262512207, + "KL/std": 31.710899353027344, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.03498149663209915, + "fcm_dpo/delta": -0.8147581815719604, + "fcm_dpo/margin": 34.44875717163086, + "fcm_dpo/q_t": 0.28912612795829773, + "grad_norm": 32.766109466552734, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.6693081855773926, + "logits/rejected": -0.681594967842102, + "logps/chosen": -67.82122802734375, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -167.8958740234375, + "loss": 0.7723, + "margin_dpo/margin_mean": 34.448753356933594, + "margin_dpo/margin_std": 36.13080978393555, + "step": 135 + }, + { + "KL/chosen_KL_mean": -19.438962936401367, + "KL/mean": -32.16922378540039, + "KL/rejected_KL_mean": -44.89949417114258, + "KL/std": 28.041908264160156, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.03454118221998215, + "fcm_dpo/delta": -0.48450008034706116, + "fcm_dpo/margin": 25.46053123474121, + "fcm_dpo/q_t": 0.342771977186203, + "grad_norm": 36.089996337890625, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.667500376701355, + "logits/rejected": -0.6432976722717285, + "logps/chosen": -84.511474609375, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -141.22073364257812, + "loss": 0.9118, + "margin_dpo/margin_mean": 25.460529327392578, + "margin_dpo/margin_std": 33.89653396606445, + "step": 136 + }, + { + "KL/chosen_KL_mean": -20.943695068359375, + "KL/mean": -39.741966247558594, + "KL/rejected_KL_mean": -58.540245056152344, + "KL/std": 38.4942741394043, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.03402690589427948, + "fcm_dpo/delta": -0.8910495042800903, + "fcm_dpo/margin": 37.596534729003906, + "fcm_dpo/q_t": 0.2910078763961792, + "grad_norm": 34.20710754394531, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.6271874308586121, + "logits/rejected": -0.6347311735153198, + "logps/chosen": -69.70281219482422, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -172.40402221679688, + "loss": 0.7998, + "margin_dpo/margin_mean": 37.596534729003906, + "margin_dpo/margin_std": 43.595252990722656, + "step": 137 + }, + { + "KL/chosen_KL_mean": -21.796977996826172, + "KL/mean": -36.171875, + "KL/rejected_KL_mean": -50.54677200317383, + "KL/std": 29.670429229736328, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.03361514210700989, + "fcm_dpo/delta": -0.5717204213142395, + "fcm_dpo/margin": 28.749794006347656, + "fcm_dpo/q_t": 0.3122233748435974, + "grad_norm": 36.481590270996094, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.6304317712783813, + "logits/rejected": -0.6085121631622314, + "logps/chosen": -82.3166275024414, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -143.74371337890625, + "loss": 0.8469, + "margin_dpo/margin_mean": 28.749794006347656, + "margin_dpo/margin_std": 31.021484375, + "step": 138 + }, + { + "KL/chosen_KL_mean": -16.36918067932129, + "KL/mean": -34.1024284362793, + "KL/rejected_KL_mean": -51.83567428588867, + "KL/std": 29.01012420654297, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.03312475234270096, + "fcm_dpo/delta": -0.7837446928024292, + "fcm_dpo/margin": 35.466487884521484, + "fcm_dpo/q_t": 0.283752977848053, + "grad_norm": 30.870824813842773, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.6461021900177002, + "logits/rejected": -0.6236545443534851, + "logps/chosen": -63.26056671142578, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -131.56365966796875, + "loss": 0.7357, + "margin_dpo/margin_mean": 35.46649169921875, + "margin_dpo/margin_std": 33.98173522949219, + "step": 139 + }, + { + "KL/chosen_KL_mean": -19.11855125427246, + "KL/mean": -35.58258056640625, + "KL/rejected_KL_mean": -52.04661178588867, + "KL/std": 32.69963073730469, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.03263962268829346, + "fcm_dpo/delta": -0.682225227355957, + "fcm_dpo/margin": 32.92806625366211, + "fcm_dpo/q_t": 0.3088887333869934, + "grad_norm": 33.12515640258789, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.6617997884750366, + "logits/rejected": -0.6407649517059326, + "logps/chosen": -78.09326171875, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -135.33071899414062, + "loss": 0.8408, + "margin_dpo/margin_mean": 32.92806625366211, + "margin_dpo/margin_std": 37.70612335205078, + "step": 140 + }, + { + "KL/chosen_KL_mean": -18.367019653320312, + "KL/mean": -34.610008239746094, + "KL/rejected_KL_mean": -50.852996826171875, + "KL/std": 31.035036087036133, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.032216668128967285, + "fcm_dpo/delta": -0.6534393429756165, + "fcm_dpo/margin": 32.48597717285156, + "fcm_dpo/q_t": 0.29906830191612244, + "grad_norm": 34.629486083984375, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.6308186054229736, + "logits/rejected": -0.6055824756622314, + "logps/chosen": -93.44268798828125, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -149.04525756835938, + "loss": 0.8167, + "margin_dpo/margin_mean": 32.48597717285156, + "margin_dpo/margin_std": 33.79344177246094, + "step": 141 + }, + { + "KL/chosen_KL_mean": -22.90228843688965, + "KL/mean": -40.82293701171875, + "KL/rejected_KL_mean": -58.74357986450195, + "KL/std": 36.93119812011719, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.031781621277332306, + "fcm_dpo/delta": -0.7471195459365845, + "fcm_dpo/margin": 35.84129333496094, + "fcm_dpo/q_t": 0.3055064380168915, + "grad_norm": 37.12379455566406, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.6237265467643738, + "logits/rejected": -0.6140986680984497, + "logps/chosen": -80.93022155761719, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -153.3258056640625, + "loss": 0.8664, + "margin_dpo/margin_mean": 35.84129333496094, + "margin_dpo/margin_std": 42.8420295715332, + "step": 142 + }, + { + "KL/chosen_KL_mean": -24.16960906982422, + "KL/mean": -36.16802215576172, + "KL/rejected_KL_mean": -48.16643524169922, + "KL/std": 31.814908981323242, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.03144429624080658, + "fcm_dpo/delta": -0.3576037883758545, + "fcm_dpo/margin": 23.996822357177734, + "fcm_dpo/q_t": 0.3577222526073456, + "grad_norm": 40.69626998901367, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.6211442947387695, + "logits/rejected": -0.5911456346511841, + "logps/chosen": -81.76605987548828, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -127.16600799560547, + "loss": 1.0086, + "margin_dpo/margin_mean": 23.9968204498291, + "margin_dpo/margin_std": 35.72144317626953, + "step": 143 + }, + { + "KL/chosen_KL_mean": -18.88149642944336, + "KL/mean": -35.042213439941406, + "KL/rejected_KL_mean": -51.20291519165039, + "KL/std": 27.641441345214844, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.03111860528588295, + "fcm_dpo/delta": -0.6120482683181763, + "fcm_dpo/margin": 32.321414947509766, + "fcm_dpo/q_t": 0.30052703619003296, + "grad_norm": 29.8353271484375, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.6261280179023743, + "logits/rejected": -0.5869371891021729, + "logps/chosen": -78.78785705566406, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -133.20318603515625, + "loss": 0.783, + "margin_dpo/margin_mean": 32.321414947509766, + "margin_dpo/margin_std": 30.421680450439453, + "step": 144 + }, + { + "KL/chosen_KL_mean": -21.75667953491211, + "KL/mean": -38.684226989746094, + "KL/rejected_KL_mean": -55.611778259277344, + "KL/std": 32.10969543457031, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.030720137059688568, + "fcm_dpo/delta": -0.6469519138336182, + "fcm_dpo/margin": 33.85509490966797, + "fcm_dpo/q_t": 0.305126428604126, + "grad_norm": 34.20407485961914, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.6173726320266724, + "logits/rejected": -0.5816211700439453, + "logps/chosen": -78.35734558105469, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -133.47808837890625, + "loss": 0.8043, + "margin_dpo/margin_mean": 33.85509490966797, + "margin_dpo/margin_std": 35.210792541503906, + "step": 145 + }, + { + "KL/chosen_KL_mean": -30.705062866210938, + "KL/mean": -44.28558349609375, + "KL/rejected_KL_mean": -57.866111755371094, + "KL/std": 32.008949279785156, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.0304176714271307, + "fcm_dpo/delta": -0.4299744963645935, + "fcm_dpo/margin": 27.161048889160156, + "fcm_dpo/q_t": 0.34171053767204285, + "grad_norm": 44.86012268066406, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.614077091217041, + "logits/rejected": -0.5655664205551147, + "logps/chosen": -96.70551300048828, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -139.56890869140625, + "loss": 0.9639, + "margin_dpo/margin_mean": 27.16104507446289, + "margin_dpo/margin_std": 36.530601501464844, + "step": 146 + }, + { + "KL/chosen_KL_mean": -23.996826171875, + "KL/mean": -42.20963668823242, + "KL/rejected_KL_mean": -60.42243576049805, + "KL/std": 34.950721740722656, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.03003515675663948, + "fcm_dpo/delta": -0.7020922899246216, + "fcm_dpo/margin": 36.42561340332031, + "fcm_dpo/q_t": 0.30501383543014526, + "grad_norm": 40.546939849853516, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.5856800079345703, + "logits/rejected": -0.5477006435394287, + "logps/chosen": -77.40231323242188, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -131.81304931640625, + "loss": 0.8245, + "margin_dpo/margin_mean": 36.42560958862305, + "margin_dpo/margin_std": 41.17435073852539, + "step": 147 + }, + { + "KL/chosen_KL_mean": -22.41992950439453, + "KL/mean": -36.62001037597656, + "KL/rejected_KL_mean": -50.82008743286133, + "KL/std": 30.988433837890625, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.02978520654141903, + "fcm_dpo/delta": -0.44947829842567444, + "fcm_dpo/margin": 28.400157928466797, + "fcm_dpo/q_t": 0.3378518521785736, + "grad_norm": 32.45388412475586, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.6462417840957642, + "logits/rejected": -0.6254644393920898, + "logps/chosen": -87.35700988769531, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -153.9139404296875, + "loss": 0.9015, + "margin_dpo/margin_mean": 28.400157928466797, + "margin_dpo/margin_std": 32.44670104980469, + "step": 148 + }, + { + "KL/chosen_KL_mean": -23.15564727783203, + "KL/mean": -41.84413146972656, + "KL/rejected_KL_mean": -60.53261184692383, + "KL/std": 32.39128494262695, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.029328342527151108, + "fcm_dpo/delta": -0.7050530314445496, + "fcm_dpo/margin": 37.37696075439453, + "fcm_dpo/q_t": 0.2920949459075928, + "grad_norm": 35.65517807006836, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.6390693187713623, + "logits/rejected": -0.6147153973579407, + "logps/chosen": -81.62940979003906, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -159.84735107421875, + "loss": 0.7704, + "margin_dpo/margin_mean": 37.37696075439453, + "margin_dpo/margin_std": 36.48619079589844, + "step": 149 + }, + { + "KL/chosen_KL_mean": -21.856395721435547, + "KL/mean": -39.75935363769531, + "KL/rejected_KL_mean": -57.66231155395508, + "KL/std": 37.55711364746094, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.028931982815265656, + "fcm_dpo/delta": -0.6441543698310852, + "fcm_dpo/margin": 35.80592346191406, + "fcm_dpo/q_t": 0.31781691312789917, + "grad_norm": 35.63197708129883, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.6231894493103027, + "logits/rejected": -0.613390326499939, + "logps/chosen": -67.56221008300781, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -141.00991821289062, + "loss": 0.8499, + "margin_dpo/margin_mean": 35.80592346191406, + "margin_dpo/margin_std": 43.49144744873047, + "step": 150 + }, + { + "KL/chosen_KL_mean": -24.731616973876953, + "KL/mean": -44.004356384277344, + "KL/rejected_KL_mean": -63.277095794677734, + "KL/std": 36.25328826904297, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.02858215942978859, + "fcm_dpo/delta": -0.7094763517379761, + "fcm_dpo/margin": 38.54548263549805, + "fcm_dpo/q_t": 0.29027602076530457, + "grad_norm": 32.182655334472656, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.6558639407157898, + "logits/rejected": -0.6159763932228088, + "logps/chosen": -95.30245208740234, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -163.74090576171875, + "loss": 0.7657, + "margin_dpo/margin_mean": 38.54548263549805, + "margin_dpo/margin_std": 37.34836959838867, + "step": 151 + }, + { + "KL/chosen_KL_mean": -21.038700103759766, + "KL/mean": -41.77024459838867, + "KL/rejected_KL_mean": -62.50178527832031, + "KL/std": 36.2904167175293, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.02817120961844921, + "fcm_dpo/delta": -0.7766570448875427, + "fcm_dpo/margin": 41.46308135986328, + "fcm_dpo/q_t": 0.2893591821193695, + "grad_norm": 30.957080841064453, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.6497496366500854, + "logits/rejected": -0.6350662112236023, + "logps/chosen": -81.20308685302734, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -168.64224243164062, + "loss": 0.7645, + "margin_dpo/margin_mean": 41.46308517456055, + "margin_dpo/margin_std": 41.2492790222168, + "step": 152 + }, + { + "KL/chosen_KL_mean": -21.354108810424805, + "KL/mean": -38.986236572265625, + "KL/rejected_KL_mean": -56.61835861206055, + "KL/std": 37.012977600097656, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.02780159004032612, + "fcm_dpo/delta": -0.5860075354576111, + "fcm_dpo/margin": 35.264251708984375, + "fcm_dpo/q_t": 0.3207632303237915, + "grad_norm": 30.601646423339844, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.6429515480995178, + "logits/rejected": -0.6301620602607727, + "logps/chosen": -77.66938781738281, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -142.27420043945312, + "loss": 0.8733, + "margin_dpo/margin_mean": 35.264251708984375, + "margin_dpo/margin_std": 42.88446807861328, + "step": 153 + }, + { + "KL/chosen_KL_mean": -25.740612030029297, + "KL/mean": -43.16389465332031, + "KL/rejected_KL_mean": -60.5871696472168, + "KL/std": 37.58220672607422, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.027507204562425613, + "fcm_dpo/delta": -0.563573956489563, + "fcm_dpo/margin": 34.8465576171875, + "fcm_dpo/q_t": 0.327609658241272, + "grad_norm": 38.05772018432617, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.6447149515151978, + "logits/rejected": -0.6309856176376343, + "logps/chosen": -88.48318481445312, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -164.83135986328125, + "loss": 0.9202, + "margin_dpo/margin_mean": 34.8465576171875, + "margin_dpo/margin_std": 43.79578399658203, + "step": 154 + }, + { + "KL/chosen_KL_mean": -20.890981674194336, + "KL/mean": -39.58171081542969, + "KL/rejected_KL_mean": -58.27244186401367, + "KL/std": 35.719573974609375, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.027167001739144325, + "fcm_dpo/delta": -0.6215222477912903, + "fcm_dpo/margin": 37.3814582824707, + "fcm_dpo/q_t": 0.3060518503189087, + "grad_norm": 30.424787521362305, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.642799973487854, + "logits/rejected": -0.6154798269271851, + "logps/chosen": -81.5441665649414, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -135.7646484375, + "loss": 0.8184, + "margin_dpo/margin_mean": 37.38145446777344, + "margin_dpo/margin_std": 39.992332458496094, + "step": 155 + }, + { + "KL/chosen_KL_mean": -30.716529846191406, + "KL/mean": -43.166847229003906, + "KL/rejected_KL_mean": -55.61716842651367, + "KL/std": 35.00094223022461, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.026953019201755524, + "fcm_dpo/delta": -0.27297720313072205, + "fcm_dpo/margin": 24.900634765625, + "fcm_dpo/q_t": 0.3701309561729431, + "grad_norm": 47.17982482910156, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.6053794622421265, + "logits/rejected": -0.5741821527481079, + "logps/chosen": -100.20840454101562, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -132.78646850585938, + "loss": 1.133, + "margin_dpo/margin_mean": 24.900630950927734, + "margin_dpo/margin_std": 46.251434326171875, + "step": 156 + }, + { + "KL/chosen_KL_mean": -28.882070541381836, + "KL/mean": -50.16608810424805, + "KL/rejected_KL_mean": -71.45011138916016, + "KL/std": 38.83407211303711, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.02660319022834301, + "fcm_dpo/delta": -0.7413443326950073, + "fcm_dpo/margin": 42.56804275512695, + "fcm_dpo/q_t": 0.29093462228775024, + "grad_norm": 35.170555114746094, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.6465293169021606, + "logits/rejected": -0.6344527006149292, + "logps/chosen": -90.25050354003906, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -179.0964813232422, + "loss": 0.7826, + "margin_dpo/margin_mean": 42.56804275512695, + "margin_dpo/margin_std": 43.04310607910156, + "step": 157 + }, + { + "KL/chosen_KL_mean": -25.599327087402344, + "KL/mean": -46.950653076171875, + "KL/rejected_KL_mean": -68.30197143554688, + "KL/std": 45.154937744140625, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.026229022070765495, + "fcm_dpo/delta": -0.7282998561859131, + "fcm_dpo/margin": 42.70265197753906, + "fcm_dpo/q_t": 0.3117641806602478, + "grad_norm": 28.284664154052734, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.6149126291275024, + "logits/rejected": -0.6111452579498291, + "logps/chosen": -83.21224975585938, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -181.99664306640625, + "loss": 0.8356, + "margin_dpo/margin_mean": 42.70265197753906, + "margin_dpo/margin_std": 52.00483703613281, + "step": 158 + }, + { + "KL/chosen_KL_mean": -28.870655059814453, + "KL/mean": -46.699745178222656, + "KL/rejected_KL_mean": -64.52882385253906, + "KL/std": 41.280426025390625, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.025942817330360413, + "fcm_dpo/delta": -0.5297616124153137, + "fcm_dpo/margin": 35.65817642211914, + "fcm_dpo/q_t": 0.3300109803676605, + "grad_norm": 35.471405029296875, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.6428400874137878, + "logits/rejected": -0.6002300977706909, + "logps/chosen": -110.43099975585938, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -153.4275360107422, + "loss": 0.8999, + "margin_dpo/margin_mean": 35.65817642211914, + "margin_dpo/margin_std": 44.819557189941406, + "step": 159 + }, + { + "KL/chosen_KL_mean": -28.709897994995117, + "KL/mean": -48.19725036621094, + "KL/rejected_KL_mean": -67.68460083007812, + "KL/std": 40.419673919677734, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.025614429265260696, + "fcm_dpo/delta": -0.6047173738479614, + "fcm_dpo/margin": 38.974700927734375, + "fcm_dpo/q_t": 0.32800424098968506, + "grad_norm": 33.28897476196289, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.6582900285720825, + "logits/rejected": -0.633819580078125, + "logps/chosen": -94.44078063964844, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -164.9024200439453, + "loss": 0.8955, + "margin_dpo/margin_mean": 38.974700927734375, + "margin_dpo/margin_std": 50.40635681152344, + "step": 160 + }, + { + "KL/chosen_KL_mean": -26.252275466918945, + "KL/mean": -44.71455383300781, + "KL/rejected_KL_mean": -63.17684555053711, + "KL/std": 37.156681060791016, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.02535148896276951, + "fcm_dpo/delta": -0.5410139560699463, + "fcm_dpo/margin": 36.92456817626953, + "fcm_dpo/q_t": 0.3276364803314209, + "grad_norm": 31.305315017700195, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.6445281505584717, + "logits/rejected": -0.6238170862197876, + "logps/chosen": -78.68875122070312, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -146.6077880859375, + "loss": 0.891, + "margin_dpo/margin_mean": 36.92456817626953, + "margin_dpo/margin_std": 44.842525482177734, + "step": 161 + }, + { + "KL/chosen_KL_mean": -24.877750396728516, + "KL/mean": -43.81897735595703, + "KL/rejected_KL_mean": -62.76021194458008, + "KL/std": 41.58441162109375, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.02502647414803505, + "fcm_dpo/delta": -0.5546809434890747, + "fcm_dpo/margin": 37.88246154785156, + "fcm_dpo/q_t": 0.3280982971191406, + "grad_norm": 28.085195541381836, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.6416307687759399, + "logits/rejected": -0.6109956502914429, + "logps/chosen": -87.48833465576172, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -152.15078735351562, + "loss": 0.8848, + "margin_dpo/margin_mean": 37.88246154785156, + "margin_dpo/margin_std": 47.10114288330078, + "step": 162 + }, + { + "KL/chosen_KL_mean": -24.786483764648438, + "KL/mean": -43.22498321533203, + "KL/rejected_KL_mean": -61.66348648071289, + "KL/std": 36.10614013671875, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.024807091802358627, + "fcm_dpo/delta": -0.5194748044013977, + "fcm_dpo/margin": 36.87699890136719, + "fcm_dpo/q_t": 0.322479784488678, + "grad_norm": 27.798776626586914, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.6384243369102478, + "logits/rejected": -0.6089353561401367, + "logps/chosen": -79.80811309814453, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -137.08169555664062, + "loss": 0.8531, + "margin_dpo/margin_mean": 36.87699890136719, + "margin_dpo/margin_std": 40.375728607177734, + "step": 163 + }, + { + "KL/chosen_KL_mean": -23.491703033447266, + "KL/mean": -44.3897705078125, + "KL/rejected_KL_mean": -65.287841796875, + "KL/std": 36.0758171081543, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.024476759135723114, + "fcm_dpo/delta": -0.6304845213890076, + "fcm_dpo/margin": 41.796142578125, + "fcm_dpo/q_t": 0.2992871105670929, + "grad_norm": 33.811378479003906, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.6135309934616089, + "logits/rejected": -0.5799993276596069, + "logps/chosen": -79.13236999511719, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -144.9524688720703, + "loss": 0.7753, + "margin_dpo/margin_mean": 41.79613494873047, + "margin_dpo/margin_std": 39.4283332824707, + "step": 164 + }, + { + "KL/chosen_KL_mean": -26.92680549621582, + "KL/mean": -41.105072021484375, + "KL/rejected_KL_mean": -55.28334045410156, + "KL/std": 31.34255599975586, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.024319054558873177, + "fcm_dpo/delta": -0.29164379835128784, + "fcm_dpo/margin": 28.35653305053711, + "fcm_dpo/q_t": 0.3585801124572754, + "grad_norm": 35.119041442871094, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.6420309543609619, + "logits/rejected": -0.5970950126647949, + "logps/chosen": -88.23749542236328, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -128.95394897460938, + "loss": 0.9751, + "margin_dpo/margin_mean": 28.35653305053711, + "margin_dpo/margin_std": 36.606605529785156, + "step": 165 + }, + { + "KL/chosen_KL_mean": -25.956722259521484, + "KL/mean": -50.594932556152344, + "KL/rejected_KL_mean": -75.233154296875, + "KL/std": 47.24444580078125, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.02398506924510002, + "fcm_dpo/delta": -0.7918073534965515, + "fcm_dpo/margin": 49.27642822265625, + "fcm_dpo/q_t": 0.2994060516357422, + "grad_norm": 27.651538848876953, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.6001812219619751, + "logits/rejected": -0.6035112142562866, + "logps/chosen": -76.94032287597656, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -173.32827758789062, + "loss": 0.7807, + "margin_dpo/margin_mean": 49.27642822265625, + "margin_dpo/margin_std": 56.34642791748047, + "step": 166 + }, + { + "KL/chosen_KL_mean": -26.515958786010742, + "KL/mean": -52.4081916809082, + "KL/rejected_KL_mean": -78.30043029785156, + "KL/std": 43.190147399902344, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.023603975772857666, + "fcm_dpo/delta": -0.8326523900032043, + "fcm_dpo/margin": 51.78446578979492, + "fcm_dpo/q_t": 0.27247390151023865, + "grad_norm": 27.97585678100586, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.5620088577270508, + "logits/rejected": -0.5581908226013184, + "logps/chosen": -76.94004821777344, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -174.3308563232422, + "loss": 0.7194, + "margin_dpo/margin_mean": 51.78446578979492, + "margin_dpo/margin_std": 46.6151123046875, + "step": 167 + }, + { + "KL/chosen_KL_mean": -28.13256072998047, + "KL/mean": -48.5062370300293, + "KL/rejected_KL_mean": -68.8799057006836, + "KL/std": 39.213836669921875, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.02330862730741501, + "fcm_dpo/delta": -0.55519700050354, + "fcm_dpo/margin": 40.747344970703125, + "fcm_dpo/q_t": 0.3147229850292206, + "grad_norm": 30.339893341064453, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.6108359098434448, + "logits/rejected": -0.5879697799682617, + "logps/chosen": -77.59538269042969, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -144.18846130371094, + "loss": 0.8262, + "margin_dpo/margin_mean": 40.747344970703125, + "margin_dpo/margin_std": 42.15760040283203, + "step": 168 + }, + { + "KL/chosen_KL_mean": -27.049694061279297, + "KL/mean": -45.38129425048828, + "KL/rejected_KL_mean": -63.712890625, + "KL/std": 41.376708984375, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.023069055750966072, + "fcm_dpo/delta": -0.4502902328968048, + "fcm_dpo/margin": 36.66320037841797, + "fcm_dpo/q_t": 0.3503206968307495, + "grad_norm": 32.7576904296875, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.6215040683746338, + "logits/rejected": -0.5975638031959534, + "logps/chosen": -86.85313415527344, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -147.0586395263672, + "loss": 0.967, + "margin_dpo/margin_mean": 36.66320037841797, + "margin_dpo/margin_std": 54.069740295410156, + "step": 169 + }, + { + "KL/chosen_KL_mean": -24.20182991027832, + "KL/mean": -47.31605529785156, + "KL/rejected_KL_mean": -70.43028259277344, + "KL/std": 39.81721878051758, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.022792475298047066, + "fcm_dpo/delta": -0.6614395380020142, + "fcm_dpo/margin": 46.22844696044922, + "fcm_dpo/q_t": 0.29807189106941223, + "grad_norm": 26.622447967529297, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.5517838001251221, + "logits/rejected": -0.5312536954879761, + "logps/chosen": -73.67359924316406, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -146.34762573242188, + "loss": 0.7923, + "margin_dpo/margin_mean": 46.22844696044922, + "margin_dpo/margin_std": 45.10525131225586, + "step": 170 + }, + { + "KL/chosen_KL_mean": -35.96885299682617, + "KL/mean": -54.98626708984375, + "KL/rejected_KL_mean": -74.0036849975586, + "KL/std": 44.122901916503906, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.02258051186800003, + "fcm_dpo/delta": -0.462841272354126, + "fcm_dpo/margin": 38.03482437133789, + "fcm_dpo/q_t": 0.3422735333442688, + "grad_norm": 37.67350769042969, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.6059920787811279, + "logits/rejected": -0.5611332654953003, + "logps/chosen": -120.46817016601562, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -183.38577270507812, + "loss": 0.9401, + "margin_dpo/margin_mean": 38.034820556640625, + "margin_dpo/margin_std": 52.570343017578125, + "step": 171 + }, + { + "KL/chosen_KL_mean": -37.53773498535156, + "KL/mean": -57.0352897644043, + "KL/rejected_KL_mean": -76.5328369140625, + "KL/std": 45.352020263671875, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.022355839610099792, + "fcm_dpo/delta": -0.47626423835754395, + "fcm_dpo/margin": 38.99510192871094, + "fcm_dpo/q_t": 0.3338128328323364, + "grad_norm": 33.96700668334961, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.6338964700698853, + "logits/rejected": -0.5965722799301147, + "logps/chosen": -106.191650390625, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -161.96951293945312, + "loss": 0.9166, + "margin_dpo/margin_mean": 38.99510192871094, + "margin_dpo/margin_std": 48.99057388305664, + "step": 172 + }, + { + "KL/chosen_KL_mean": -32.11933135986328, + "KL/mean": -52.062782287597656, + "KL/rejected_KL_mean": -72.00623321533203, + "KL/std": 41.18177795410156, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.02216143161058426, + "fcm_dpo/delta": -0.48810991644859314, + "fcm_dpo/margin": 39.88690185546875, + "fcm_dpo/q_t": 0.3256625533103943, + "grad_norm": 32.12683868408203, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.6493266224861145, + "logits/rejected": -0.6063686609268188, + "logps/chosen": -95.17021179199219, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -150.69015502929688, + "loss": 0.8559, + "margin_dpo/margin_mean": 39.88690185546875, + "margin_dpo/margin_std": 43.12532043457031, + "step": 173 + }, + { + "KL/chosen_KL_mean": -34.28569793701172, + "KL/mean": -55.415374755859375, + "KL/rejected_KL_mean": -76.54505920410156, + "KL/std": 45.56023025512695, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.02194075658917427, + "fcm_dpo/delta": -0.5318416953086853, + "fcm_dpo/margin": 42.25935363769531, + "fcm_dpo/q_t": 0.32547903060913086, + "grad_norm": 40.07661819458008, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.5506089925765991, + "logits/rejected": -0.5634474754333496, + "logps/chosen": -87.64866638183594, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -178.45626831054688, + "loss": 0.8748, + "margin_dpo/margin_mean": 42.25935363769531, + "margin_dpo/margin_std": 48.72834014892578, + "step": 174 + }, + { + "KL/chosen_KL_mean": -33.68284606933594, + "KL/mean": -63.342628479003906, + "KL/rejected_KL_mean": -93.00241088867188, + "KL/std": 52.50614547729492, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.021571001037955284, + "fcm_dpo/delta": -0.8911694288253784, + "fcm_dpo/margin": 59.319557189941406, + "fcm_dpo/q_t": 0.26907652616500854, + "grad_norm": 38.66853713989258, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.4975181221961975, + "logits/rejected": -0.48338231444358826, + "logps/chosen": -79.10061645507812, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -182.50820922851562, + "loss": 0.6898, + "margin_dpo/margin_mean": 59.319557189941406, + "margin_dpo/margin_std": 54.04429244995117, + "step": 175 + }, + { + "KL/chosen_KL_mean": -34.378013610839844, + "KL/mean": -59.340576171875, + "KL/rejected_KL_mean": -84.30314636230469, + "KL/std": 47.96424102783203, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.02127264440059662, + "fcm_dpo/delta": -0.6689236164093018, + "fcm_dpo/margin": 49.92512512207031, + "fcm_dpo/q_t": 0.3060389459133148, + "grad_norm": 49.96451187133789, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.5411580801010132, + "logits/rejected": -0.5494238138198853, + "logps/chosen": -84.83085632324219, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -179.8621063232422, + "loss": 0.797, + "margin_dpo/margin_mean": 49.92512512207031, + "margin_dpo/margin_std": 53.52428436279297, + "step": 176 + }, + { + "KL/chosen_KL_mean": -41.88209533691406, + "KL/mean": -67.57048034667969, + "KL/rejected_KL_mean": -93.25885772705078, + "KL/std": 55.000179290771484, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.02098482847213745, + "fcm_dpo/delta": -0.685294508934021, + "fcm_dpo/margin": 51.37677001953125, + "fcm_dpo/q_t": 0.30864325165748596, + "grad_norm": 36.14768600463867, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.5516154766082764, + "logits/rejected": -0.5299651622772217, + "logps/chosen": -103.09855651855469, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -189.15264892578125, + "loss": 0.8214, + "margin_dpo/margin_mean": 51.37677001953125, + "margin_dpo/margin_std": 58.066383361816406, + "step": 177 + }, + { + "KL/chosen_KL_mean": -46.72706604003906, + "KL/mean": -78.73434448242188, + "KL/rejected_KL_mean": -110.74161529541016, + "KL/std": 64.29850006103516, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.02060597389936447, + "fcm_dpo/delta": -0.9318315982818604, + "fcm_dpo/margin": 64.01455688476562, + "fcm_dpo/q_t": 0.27863284945487976, + "grad_norm": 35.291893005371094, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.5071342587471008, + "logits/rejected": -0.5141210556030273, + "logps/chosen": -104.99185180664062, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -216.10694885253906, + "loss": 0.7491, + "margin_dpo/margin_mean": 64.0145492553711, + "margin_dpo/margin_std": 68.846435546875, + "step": 178 + }, + { + "KL/chosen_KL_mean": -44.31977081298828, + "KL/mean": -71.11123657226562, + "KL/rejected_KL_mean": -97.90270233154297, + "KL/std": 55.558712005615234, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.020287491381168365, + "fcm_dpo/delta": -0.6954261064529419, + "fcm_dpo/margin": 53.58293151855469, + "fcm_dpo/q_t": 0.31118741631507874, + "grad_norm": 40.223323822021484, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.5324760675430298, + "logits/rejected": -0.5225632190704346, + "logps/chosen": -105.37809753417969, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -188.43052673339844, + "loss": 0.8991, + "margin_dpo/margin_mean": 53.58293151855469, + "margin_dpo/margin_std": 66.26374816894531, + "step": 179 + }, + { + "KL/chosen_KL_mean": -37.499053955078125, + "KL/mean": -66.82682037353516, + "KL/rejected_KL_mean": -96.15459442138672, + "KL/std": 50.33916473388672, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.019992802292108536, + "fcm_dpo/delta": -0.7821571826934814, + "fcm_dpo/margin": 58.65553283691406, + "fcm_dpo/q_t": 0.2883349657058716, + "grad_norm": 42.83848190307617, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.5499392747879028, + "logits/rejected": -0.5351977348327637, + "logps/chosen": -91.8417739868164, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -194.36642456054688, + "loss": 0.7723, + "margin_dpo/margin_mean": 58.65553283691406, + "margin_dpo/margin_std": 59.56299591064453, + "step": 180 + }, + { + "KL/chosen_KL_mean": -32.0028190612793, + "KL/mean": -51.6584587097168, + "KL/rejected_KL_mean": -71.31410217285156, + "KL/std": 45.07472229003906, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.01981274038553238, + "fcm_dpo/delta": -0.3817533254623413, + "fcm_dpo/margin": 39.3112907409668, + "fcm_dpo/q_t": 0.3500908613204956, + "grad_norm": 31.20648956298828, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.5727461576461792, + "logits/rejected": -0.5383597612380981, + "logps/chosen": -87.00328063964844, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -132.97027587890625, + "loss": 0.9616, + "margin_dpo/margin_mean": 39.3112907409668, + "margin_dpo/margin_std": 53.121646881103516, + "step": 181 + }, + { + "KL/chosen_KL_mean": -32.192604064941406, + "KL/mean": -63.41984176635742, + "KL/rejected_KL_mean": -94.64707946777344, + "KL/std": 57.1092643737793, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.019511841237545013, + "fcm_dpo/delta": -0.8293993473052979, + "fcm_dpo/margin": 62.45448303222656, + "fcm_dpo/q_t": 0.28383222222328186, + "grad_norm": 33.97060012817383, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.5092509984970093, + "logits/rejected": -0.5052568912506104, + "logps/chosen": -73.30045318603516, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -184.16867065429688, + "loss": 0.7931, + "margin_dpo/margin_mean": 62.45448303222656, + "margin_dpo/margin_std": 65.73297119140625, + "step": 182 + }, + { + "KL/chosen_KL_mean": -46.7093505859375, + "KL/mean": -67.55374145507812, + "KL/rejected_KL_mean": -88.39813232421875, + "KL/std": 44.44742965698242, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.01928892359137535, + "fcm_dpo/delta": -0.4089146852493286, + "fcm_dpo/margin": 41.68878173828125, + "fcm_dpo/q_t": 0.34354251623153687, + "grad_norm": 34.32046127319336, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.5425801873207092, + "logits/rejected": -0.5084269046783447, + "logps/chosen": -104.23391723632812, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -164.3738555908203, + "loss": 0.9272, + "margin_dpo/margin_mean": 41.68878173828125, + "margin_dpo/margin_std": 47.48735046386719, + "step": 183 + }, + { + "KL/chosen_KL_mean": -34.87854766845703, + "KL/mean": -53.72959899902344, + "KL/rejected_KL_mean": -72.5806655883789, + "KL/std": 42.14961242675781, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.019207663834095, + "fcm_dpo/delta": -0.32640156149864197, + "fcm_dpo/margin": 37.702117919921875, + "fcm_dpo/q_t": 0.3587670922279358, + "grad_norm": 29.850723266601562, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.591180682182312, + "logits/rejected": -0.5764235854148865, + "logps/chosen": -93.42350006103516, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -149.2147216796875, + "loss": 0.9602, + "margin_dpo/margin_mean": 37.702117919921875, + "margin_dpo/margin_std": 48.818450927734375, + "step": 184 + }, + { + "KL/chosen_KL_mean": -39.895015716552734, + "KL/mean": -55.8941650390625, + "KL/rejected_KL_mean": -71.893310546875, + "KL/std": 44.797210693359375, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.01909426599740982, + "fcm_dpo/delta": -0.21235749125480652, + "fcm_dpo/margin": 31.998287200927734, + "fcm_dpo/q_t": 0.3808383643627167, + "grad_norm": 33.721702575683594, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.5846891403198242, + "logits/rejected": -0.5383951663970947, + "logps/chosen": -101.92086791992188, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -145.65591430664062, + "loss": 1.0554, + "margin_dpo/margin_mean": 31.998291015625, + "margin_dpo/margin_std": 51.16020202636719, + "step": 185 + }, + { + "KL/chosen_KL_mean": -37.378414154052734, + "KL/mean": -64.8458251953125, + "KL/rejected_KL_mean": -92.31324768066406, + "KL/std": 50.69861602783203, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.018892861902713776, + "fcm_dpo/delta": -0.644684374332428, + "fcm_dpo/margin": 54.93482971191406, + "fcm_dpo/q_t": 0.29887640476226807, + "grad_norm": 33.86766052246094, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.5974804759025574, + "logits/rejected": -0.549450159072876, + "logps/chosen": -106.73188781738281, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -180.38571166992188, + "loss": 0.7728, + "margin_dpo/margin_mean": 54.93482971191406, + "margin_dpo/margin_std": 52.707542419433594, + "step": 186 + }, + { + "KL/chosen_KL_mean": -38.300201416015625, + "KL/mean": -59.789634704589844, + "KL/rejected_KL_mean": -81.27906799316406, + "KL/std": 48.23352813720703, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.01874019205570221, + "fcm_dpo/delta": -0.40851715207099915, + "fcm_dpo/margin": 42.9788703918457, + "fcm_dpo/q_t": 0.3437439501285553, + "grad_norm": 33.6903076171875, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.5507239103317261, + "logits/rejected": -0.5455770492553711, + "logps/chosen": -91.05665588378906, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -163.2481689453125, + "loss": 0.9349, + "margin_dpo/margin_mean": 42.97887420654297, + "margin_dpo/margin_std": 53.59326934814453, + "step": 187 + }, + { + "KL/chosen_KL_mean": -33.11475372314453, + "KL/mean": -60.31569290161133, + "KL/rejected_KL_mean": -87.51663208007812, + "KL/std": 53.4353141784668, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.018511701375246048, + "fcm_dpo/delta": -0.6132241487503052, + "fcm_dpo/margin": 54.401878356933594, + "fcm_dpo/q_t": 0.31008559465408325, + "grad_norm": 44.41658020019531, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.5466028451919556, + "logits/rejected": -0.5393306016921997, + "logps/chosen": -82.53024291992188, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -177.05706787109375, + "loss": 0.8197, + "margin_dpo/margin_mean": 54.40187072753906, + "margin_dpo/margin_std": 57.68813705444336, + "step": 188 + }, + { + "KL/chosen_KL_mean": -39.16114044189453, + "KL/mean": -62.931758880615234, + "KL/rejected_KL_mean": -86.70237731933594, + "KL/std": 50.829307556152344, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.018328163772821426, + "fcm_dpo/delta": -0.47537046670913696, + "fcm_dpo/margin": 47.541236877441406, + "fcm_dpo/q_t": 0.3356119394302368, + "grad_norm": 35.31798553466797, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.531764030456543, + "logits/rejected": -0.4874729514122009, + "logps/chosen": -91.56010437011719, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -158.86972045898438, + "loss": 0.908, + "margin_dpo/margin_mean": 47.54124069213867, + "margin_dpo/margin_std": 59.69377899169922, + "step": 189 + }, + { + "KL/chosen_KL_mean": -40.30663299560547, + "KL/mean": -66.10369110107422, + "KL/rejected_KL_mean": -91.9007568359375, + "KL/std": 52.72616195678711, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.018157871440052986, + "fcm_dpo/delta": -0.5415529608726501, + "fcm_dpo/margin": 51.594120025634766, + "fcm_dpo/q_t": 0.32628098130226135, + "grad_norm": 33.406375885009766, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.5960294604301453, + "logits/rejected": -0.5646297931671143, + "logps/chosen": -104.98970031738281, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -194.4512939453125, + "loss": 0.8874, + "margin_dpo/margin_mean": 51.5941162109375, + "margin_dpo/margin_std": 60.433231353759766, + "step": 190 + }, + { + "KL/chosen_KL_mean": -34.816123962402344, + "KL/mean": -71.40798950195312, + "KL/rejected_KL_mean": -107.9998550415039, + "KL/std": 66.14059448242188, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.017826292663812637, + "fcm_dpo/delta": -0.9171139001846313, + "fcm_dpo/margin": 73.18373107910156, + "fcm_dpo/q_t": 0.26971107721328735, + "grad_norm": 28.79018783569336, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.5394096970558167, + "logits/rejected": -0.5245342254638672, + "logps/chosen": -103.47499084472656, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -218.1395263671875, + "loss": 0.6903, + "margin_dpo/margin_mean": 73.18373107910156, + "margin_dpo/margin_std": 70.1646728515625, + "step": 191 + }, + { + "KL/chosen_KL_mean": -48.116294860839844, + "KL/mean": -75.11026000976562, + "KL/rejected_KL_mean": -102.104248046875, + "KL/std": 56.96019744873047, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.017613738775253296, + "fcm_dpo/delta": -0.5560271143913269, + "fcm_dpo/margin": 53.987945556640625, + "fcm_dpo/q_t": 0.3247717320919037, + "grad_norm": 34.394615173339844, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.5471304059028625, + "logits/rejected": -0.5359860062599182, + "logps/chosen": -117.84320831298828, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -205.42559814453125, + "loss": 0.8666, + "margin_dpo/margin_mean": 53.987945556640625, + "margin_dpo/margin_std": 63.478248596191406, + "step": 192 + }, + { + "KL/chosen_KL_mean": -50.61434555053711, + "KL/mean": -66.77139282226562, + "KL/rejected_KL_mean": -82.92842864990234, + "KL/std": 46.935752868652344, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.017528578639030457, + "fcm_dpo/delta": -0.1673727035522461, + "fcm_dpo/margin": 32.31407928466797, + "fcm_dpo/q_t": 0.3779892325401306, + "grad_norm": 46.32831954956055, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.5371495485305786, + "logits/rejected": -0.4979764521121979, + "logps/chosen": -110.80484008789062, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -159.3359832763672, + "loss": 1.1055, + "margin_dpo/margin_mean": 32.31407928466797, + "margin_dpo/margin_std": 55.97624969482422, + "step": 193 + }, + { + "KL/chosen_KL_mean": -31.121389389038086, + "KL/mean": -56.82708740234375, + "KL/rejected_KL_mean": -82.53279113769531, + "KL/std": 47.063621520996094, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.017367277294397354, + "fcm_dpo/delta": -0.4973655939102173, + "fcm_dpo/margin": 51.41139602661133, + "fcm_dpo/q_t": 0.31907355785369873, + "grad_norm": 29.248727798461914, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.47672080993652344, + "logits/rejected": -0.4526366591453552, + "logps/chosen": -68.96176147460938, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -143.2175750732422, + "loss": 0.8256, + "margin_dpo/margin_mean": 51.411399841308594, + "margin_dpo/margin_std": 49.374176025390625, + "step": 194 + }, + { + "KL/chosen_KL_mean": -52.05164337158203, + "KL/mean": -78.95748138427734, + "KL/rejected_KL_mean": -105.86331939697266, + "KL/std": 52.807838439941406, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.017184413969516754, + "fcm_dpo/delta": -0.5297634601593018, + "fcm_dpo/margin": 53.811676025390625, + "fcm_dpo/q_t": 0.3161800503730774, + "grad_norm": 36.198482513427734, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.4681174159049988, + "logits/rejected": -0.479825496673584, + "logps/chosen": -106.94320678710938, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -202.63427734375, + "loss": 0.8219, + "margin_dpo/margin_mean": 53.811676025390625, + "margin_dpo/margin_std": 55.032249450683594, + "step": 195 + }, + { + "KL/chosen_KL_mean": -40.59221649169922, + "KL/mean": -72.35244750976562, + "KL/rejected_KL_mean": -104.1126937866211, + "KL/std": 60.947166442871094, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.01695842109620571, + "fcm_dpo/delta": -0.6850365996360779, + "fcm_dpo/margin": 63.520469665527344, + "fcm_dpo/q_t": 0.30276960134506226, + "grad_norm": 33.90129089355469, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.4985538125038147, + "logits/rejected": -0.4695102274417877, + "logps/chosen": -93.83746337890625, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -180.1656494140625, + "loss": 0.8061, + "margin_dpo/margin_mean": 63.520469665527344, + "margin_dpo/margin_std": 66.7315902709961, + "step": 196 + }, + { + "KL/chosen_KL_mean": -42.02714538574219, + "KL/mean": -67.51123046875, + "KL/rejected_KL_mean": -92.99533081054688, + "KL/std": 51.72228240966797, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.01680205762386322, + "fcm_dpo/delta": -0.46013152599334717, + "fcm_dpo/margin": 50.968177795410156, + "fcm_dpo/q_t": 0.3302619457244873, + "grad_norm": 31.701984405517578, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.4703928530216217, + "logits/rejected": -0.4579269289970398, + "logps/chosen": -102.44747924804688, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -170.20423889160156, + "loss": 0.8638, + "margin_dpo/margin_mean": 50.968177795410156, + "margin_dpo/margin_std": 55.38848114013672, + "step": 197 + }, + { + "KL/chosen_KL_mean": -46.086952209472656, + "KL/mean": -75.67738342285156, + "KL/rejected_KL_mean": -105.26781463623047, + "KL/std": 61.393341064453125, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.01661756820976734, + "fcm_dpo/delta": -0.5889589786529541, + "fcm_dpo/margin": 59.18085861206055, + "fcm_dpo/q_t": 0.3176442086696625, + "grad_norm": 30.10756492614746, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.5225601196289062, + "logits/rejected": -0.5224871635437012, + "logps/chosen": -101.12313842773438, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -202.51107788085938, + "loss": 0.8319, + "margin_dpo/margin_mean": 59.18086242675781, + "margin_dpo/margin_std": 68.20362091064453, + "step": 198 + }, + { + "KL/chosen_KL_mean": -45.904319763183594, + "KL/mean": -74.5548095703125, + "KL/rejected_KL_mean": -103.20530700683594, + "KL/std": 57.711090087890625, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.016409363597631454, + "fcm_dpo/delta": -0.5460337996482849, + "fcm_dpo/margin": 57.300987243652344, + "fcm_dpo/q_t": 0.32322460412979126, + "grad_norm": 28.70806121826172, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.5098904967308044, + "logits/rejected": -0.4957270920276642, + "logps/chosen": -102.73314666748047, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -187.853515625, + "loss": 0.8747, + "margin_dpo/margin_mean": 57.300987243652344, + "margin_dpo/margin_std": 66.47672271728516, + "step": 199 + }, + { + "KL/chosen_KL_mean": -47.09435272216797, + "KL/mean": -78.50828552246094, + "KL/rejected_KL_mean": -109.9222183227539, + "KL/std": 64.87405395507812, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.016217276453971863, + "fcm_dpo/delta": -0.6256648898124695, + "fcm_dpo/margin": 62.82786560058594, + "fcm_dpo/q_t": 0.3162272572517395, + "grad_norm": 33.296630859375, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.49233245849609375, + "logits/rejected": -0.46740299463272095, + "logps/chosen": -100.16141510009766, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -190.53065490722656, + "loss": 0.8636, + "margin_dpo/margin_mean": 62.82786560058594, + "margin_dpo/margin_std": 75.18763732910156, + "step": 200 + }, + { + "KL/chosen_KL_mean": -46.16725540161133, + "KL/mean": -78.2830581665039, + "KL/rejected_KL_mean": -110.39886474609375, + "KL/std": 64.71347045898438, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.016018882393836975, + "fcm_dpo/delta": -0.6355843544006348, + "fcm_dpo/margin": 64.23160552978516, + "fcm_dpo/q_t": 0.31057775020599365, + "grad_norm": 33.702178955078125, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.4950675964355469, + "logits/rejected": -0.4797266125679016, + "logps/chosen": -121.56947326660156, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -225.20709228515625, + "loss": 0.8403, + "margin_dpo/margin_mean": 64.23160552978516, + "margin_dpo/margin_std": 73.48382568359375, + "step": 201 + }, + { + "KL/chosen_KL_mean": -51.50401306152344, + "KL/mean": -74.23617553710938, + "KL/rejected_KL_mean": -96.96835327148438, + "KL/std": 57.23125457763672, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.015895845368504524, + "fcm_dpo/delta": -0.3249945044517517, + "fcm_dpo/margin": 45.464332580566406, + "fcm_dpo/q_t": 0.36251670122146606, + "grad_norm": 35.321651458740234, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.45440933108329773, + "logits/rejected": -0.45531147718429565, + "logps/chosen": -101.60533142089844, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -183.95338439941406, + "loss": 1.0272, + "margin_dpo/margin_mean": 45.464332580566406, + "margin_dpo/margin_std": 69.42536926269531, + "step": 202 + }, + { + "KL/chosen_KL_mean": -47.888099670410156, + "KL/mean": -75.02025604248047, + "KL/rejected_KL_mean": -102.15240478515625, + "KL/std": 58.18334197998047, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.01575688272714615, + "fcm_dpo/delta": -0.4590129256248474, + "fcm_dpo/margin": 54.264312744140625, + "fcm_dpo/q_t": 0.33164793252944946, + "grad_norm": 35.848411560058594, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.5049876570701599, + "logits/rejected": -0.4844914674758911, + "logps/chosen": -108.49779510498047, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -188.04837036132812, + "loss": 0.8714, + "margin_dpo/margin_mean": 54.264312744140625, + "margin_dpo/margin_std": 60.59400939941406, + "step": 203 + }, + { + "KL/chosen_KL_mean": -57.63642120361328, + "KL/mean": -83.66537475585938, + "KL/rejected_KL_mean": -109.69432067871094, + "KL/std": 65.74458312988281, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.015622062608599663, + "fcm_dpo/delta": -0.41670918464660645, + "fcm_dpo/margin": 52.05790710449219, + "fcm_dpo/q_t": 0.35108113288879395, + "grad_norm": 43.6108512878418, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.4982585310935974, + "logits/rejected": -0.4530515670776367, + "logps/chosen": -137.80137634277344, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -197.39022827148438, + "loss": 0.9475, + "margin_dpo/margin_mean": 52.05790710449219, + "margin_dpo/margin_std": 71.62932586669922, + "step": 204 + }, + { + "KL/chosen_KL_mean": -58.564178466796875, + "KL/mean": -90.26313781738281, + "KL/rejected_KL_mean": -121.96208953857422, + "KL/std": 68.35162353515625, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.015434244647622108, + "fcm_dpo/delta": -0.5854172706604004, + "fcm_dpo/margin": 63.39790725708008, + "fcm_dpo/q_t": 0.3134157657623291, + "grad_norm": 33.84364700317383, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.4288904368877411, + "logits/rejected": -0.3909919261932373, + "logps/chosen": -117.94891357421875, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -207.08714294433594, + "loss": 0.8694, + "margin_dpo/margin_mean": 63.39790725708008, + "margin_dpo/margin_std": 69.564208984375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -57.4029541015625, + "KL/mean": -93.16062927246094, + "KL/rejected_KL_mean": -128.91830444335938, + "KL/std": 68.78971099853516, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.015242546796798706, + "fcm_dpo/delta": -0.6980785131454468, + "fcm_dpo/margin": 71.51534271240234, + "fcm_dpo/q_t": 0.2959573268890381, + "grad_norm": 44.374515533447266, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.38661372661590576, + "logits/rejected": -0.3897287845611572, + "logps/chosen": -104.3674545288086, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -227.87179565429688, + "loss": 0.8016, + "margin_dpo/margin_mean": 71.51535034179688, + "margin_dpo/margin_std": 73.881103515625, + "step": 206 + }, + { + "KL/chosen_KL_mean": -43.565460205078125, + "KL/mean": -83.27995300292969, + "KL/rejected_KL_mean": -122.99443054199219, + "KL/std": 73.8983154296875, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.015017333440482616, + "fcm_dpo/delta": -0.8022794723510742, + "fcm_dpo/margin": 79.428955078125, + "fcm_dpo/q_t": 0.2901962101459503, + "grad_norm": 33.59035110473633, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.51853346824646, + "logits/rejected": -0.49110108613967896, + "logps/chosen": -99.62171936035156, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -207.44223022460938, + "loss": 0.7534, + "margin_dpo/margin_mean": 79.428955078125, + "margin_dpo/margin_std": 82.80197143554688, + "step": 207 + }, + { + "KL/chosen_KL_mean": -64.4532699584961, + "KL/mean": -96.2532958984375, + "KL/rejected_KL_mean": -128.05331420898438, + "KL/std": 68.32240295410156, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.014828778803348541, + "fcm_dpo/delta": -0.5484308004379272, + "fcm_dpo/margin": 63.60005187988281, + "fcm_dpo/q_t": 0.3209478259086609, + "grad_norm": 39.83061218261719, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.5061453580856323, + "logits/rejected": -0.4840332865715027, + "logps/chosen": -131.5208740234375, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -222.3402099609375, + "loss": 0.8703, + "margin_dpo/margin_mean": 63.60005187988281, + "margin_dpo/margin_std": 74.10440063476562, + "step": 208 + }, + { + "KL/chosen_KL_mean": -52.41194534301758, + "KL/mean": -80.66610717773438, + "KL/rejected_KL_mean": -108.92027282714844, + "KL/std": 57.79236602783203, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.014692675322294235, + "fcm_dpo/delta": -0.43396198749542236, + "fcm_dpo/margin": 56.508323669433594, + "fcm_dpo/q_t": 0.33858901262283325, + "grad_norm": 34.457672119140625, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.47483745217323303, + "logits/rejected": -0.4589829444885254, + "logps/chosen": -108.59364318847656, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -189.86178588867188, + "loss": 0.9222, + "margin_dpo/margin_mean": 56.508323669433594, + "margin_dpo/margin_std": 68.15226745605469, + "step": 209 + }, + { + "KL/chosen_KL_mean": -46.836769104003906, + "KL/mean": -77.15861511230469, + "KL/rejected_KL_mean": -107.48045349121094, + "KL/std": 56.17525863647461, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.014558359980583191, + "fcm_dpo/delta": -0.4871372580528259, + "fcm_dpo/margin": 60.64369201660156, + "fcm_dpo/q_t": 0.32445818185806274, + "grad_norm": 30.47566795349121, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.4821133613586426, + "logits/rejected": -0.46310529112815857, + "logps/chosen": -93.20858764648438, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -184.16207885742188, + "loss": 0.8456, + "margin_dpo/margin_mean": 60.64369201660156, + "margin_dpo/margin_std": 62.58555603027344, + "step": 210 + }, + { + "KL/chosen_KL_mean": -68.47840881347656, + "KL/mean": -93.94920349121094, + "KL/rejected_KL_mean": -119.41999816894531, + "KL/std": 58.32376480102539, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.014445479959249496, + "fcm_dpo/delta": -0.3384362459182739, + "fcm_dpo/margin": 50.941593170166016, + "fcm_dpo/q_t": 0.3547877371311188, + "grad_norm": 38.224822998046875, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.5101429224014282, + "logits/rejected": -0.4577806890010834, + "logps/chosen": -147.4107666015625, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -206.24099731445312, + "loss": 0.9619, + "margin_dpo/margin_mean": 50.94158935546875, + "margin_dpo/margin_std": 65.62732696533203, + "step": 211 + }, + { + "KL/chosen_KL_mean": -54.330657958984375, + "KL/mean": -91.82683563232422, + "KL/rejected_KL_mean": -129.32302856445312, + "KL/std": 72.2373275756836, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.01425769180059433, + "fcm_dpo/delta": -0.6778258681297302, + "fcm_dpo/margin": 74.99236297607422, + "fcm_dpo/q_t": 0.30320441722869873, + "grad_norm": 26.950992584228516, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.5148077011108398, + "logits/rejected": -0.49970927834510803, + "logps/chosen": -112.52767181396484, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -232.38087463378906, + "loss": 0.8114, + "margin_dpo/margin_mean": 74.99236297607422, + "margin_dpo/margin_std": 75.5556411743164, + "step": 212 + }, + { + "KL/chosen_KL_mean": -49.063575744628906, + "KL/mean": -85.65653991699219, + "KL/rejected_KL_mean": -122.24949645996094, + "KL/std": 63.127166748046875, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.014083405956625938, + "fcm_dpo/delta": -0.6378830671310425, + "fcm_dpo/margin": 73.18592834472656, + "fcm_dpo/q_t": 0.30264830589294434, + "grad_norm": 28.00612449645996, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.5043731331825256, + "logits/rejected": -0.4619431793689728, + "logps/chosen": -116.57628631591797, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -216.16421508789062, + "loss": 0.8, + "margin_dpo/margin_mean": 73.18592071533203, + "margin_dpo/margin_std": 73.15200805664062, + "step": 213 + }, + { + "KL/chosen_KL_mean": -48.69452667236328, + "KL/mean": -78.96730041503906, + "KL/rejected_KL_mean": -109.24006652832031, + "KL/std": 64.6525650024414, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.013958265073597431, + "fcm_dpo/delta": -0.4488096237182617, + "fcm_dpo/margin": 60.545536041259766, + "fcm_dpo/q_t": 0.3374096751213074, + "grad_norm": 32.47929763793945, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.4716220796108246, + "logits/rejected": -0.47583937644958496, + "logps/chosen": -90.2994155883789, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -186.75747680664062, + "loss": 0.9073, + "margin_dpo/margin_mean": 60.54553985595703, + "margin_dpo/margin_std": 73.65911865234375, + "step": 214 + }, + { + "KL/chosen_KL_mean": -52.654701232910156, + "KL/mean": -85.257568359375, + "KL/rejected_KL_mean": -117.8604507446289, + "KL/std": 62.362545013427734, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.013811732642352581, + "fcm_dpo/delta": -0.5054513812065125, + "fcm_dpo/margin": 65.20574188232422, + "fcm_dpo/q_t": 0.3219216763973236, + "grad_norm": 27.356969833374023, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.49429577589035034, + "logits/rejected": -0.47771984338760376, + "logps/chosen": -105.93396759033203, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -207.82510375976562, + "loss": 0.8437, + "margin_dpo/margin_mean": 65.20574188232422, + "margin_dpo/margin_std": 68.63744354248047, + "step": 215 + }, + { + "KL/chosen_KL_mean": -55.992122650146484, + "KL/mean": -90.47587585449219, + "KL/rejected_KL_mean": -124.95962524414062, + "KL/std": 65.91754150390625, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.013682305812835693, + "fcm_dpo/delta": -0.5484941005706787, + "fcm_dpo/margin": 68.9675064086914, + "fcm_dpo/q_t": 0.32259291410446167, + "grad_norm": 33.01445007324219, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.4938111901283264, + "logits/rejected": -0.47213661670684814, + "logps/chosen": -104.87991333007812, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -202.15855407714844, + "loss": 0.8764, + "margin_dpo/margin_mean": 68.9675064086914, + "margin_dpo/margin_std": 78.93191528320312, + "step": 216 + }, + { + "KL/chosen_KL_mean": -52.87226867675781, + "KL/mean": -90.88249206542969, + "KL/rejected_KL_mean": -128.8927001953125, + "KL/std": 65.4497299194336, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.01350276730954647, + "fcm_dpo/delta": -0.6330698132514954, + "fcm_dpo/margin": 76.02044677734375, + "fcm_dpo/q_t": 0.29809918999671936, + "grad_norm": 28.874114990234375, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.5081589221954346, + "logits/rejected": -0.49316224455833435, + "logps/chosen": -102.71757507324219, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -228.97103881835938, + "loss": 0.7681, + "margin_dpo/margin_mean": 76.02043914794922, + "margin_dpo/margin_std": 69.90164184570312, + "step": 217 + }, + { + "KL/chosen_KL_mean": -56.74638748168945, + "KL/mean": -88.20991516113281, + "KL/rejected_KL_mean": -119.6734390258789, + "KL/std": 64.20523071289062, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.013370934873819351, + "fcm_dpo/delta": -0.4451802968978882, + "fcm_dpo/margin": 62.92705535888672, + "fcm_dpo/q_t": 0.33391833305358887, + "grad_norm": 28.70237922668457, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.4731810688972473, + "logits/rejected": -0.4529051184654236, + "logps/chosen": -115.32306671142578, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -207.51983642578125, + "loss": 0.8901, + "margin_dpo/margin_mean": 62.92705154418945, + "margin_dpo/margin_std": 72.0788345336914, + "step": 218 + }, + { + "KL/chosen_KL_mean": -59.74544906616211, + "KL/mean": -86.31004333496094, + "KL/rejected_KL_mean": -112.87462615966797, + "KL/std": 61.03803634643555, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.013287264853715897, + "fcm_dpo/delta": -0.30801448225975037, + "fcm_dpo/margin": 53.129173278808594, + "fcm_dpo/q_t": 0.3569221496582031, + "grad_norm": 32.149879455566406, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.49771422147750854, + "logits/rejected": -0.4766976237297058, + "logps/chosen": -120.82930755615234, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -198.70504760742188, + "loss": 0.9684, + "margin_dpo/margin_mean": 53.129173278808594, + "margin_dpo/margin_std": 68.73921203613281, + "step": 219 + }, + { + "KL/chosen_KL_mean": -66.4417724609375, + "KL/mean": -91.07945251464844, + "KL/rejected_KL_mean": -115.71713256835938, + "KL/std": 55.46550750732422, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.013200750574469566, + "fcm_dpo/delta": -0.25239700078964233, + "fcm_dpo/margin": 49.275352478027344, + "fcm_dpo/q_t": 0.3582019507884979, + "grad_norm": 31.212749481201172, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.4796912372112274, + "logits/rejected": -0.44858330488204956, + "logps/chosen": -136.47305297851562, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -203.40264892578125, + "loss": 0.9431, + "margin_dpo/margin_mean": 49.275352478027344, + "margin_dpo/margin_std": 51.30287170410156, + "step": 220 + }, + { + "KL/chosen_KL_mean": -67.75740051269531, + "KL/mean": -113.33877563476562, + "KL/rejected_KL_mean": -158.92015075683594, + "KL/std": 77.45755767822266, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.01304638385772705, + "fcm_dpo/delta": -0.7983723878860474, + "fcm_dpo/margin": 91.16275024414062, + "fcm_dpo/q_t": 0.26999005675315857, + "grad_norm": 30.836456298828125, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.4370056986808777, + "logits/rejected": -0.4412776827812195, + "logps/chosen": -119.912109375, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -265.3878173828125, + "loss": 0.6906, + "margin_dpo/margin_mean": 91.16275024414062, + "margin_dpo/margin_std": 72.8553695678711, + "step": 221 + }, + { + "KL/chosen_KL_mean": -70.8458023071289, + "KL/mean": -106.4292221069336, + "KL/rejected_KL_mean": -142.01263427734375, + "KL/std": 70.39703369140625, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.01289133820682764, + "fcm_dpo/delta": -0.5219833850860596, + "fcm_dpo/margin": 71.16682434082031, + "fcm_dpo/q_t": 0.31928715109825134, + "grad_norm": 33.059791564941406, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.4564954936504364, + "logits/rejected": -0.4395928978919983, + "logps/chosen": -131.81689453125, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -242.0137939453125, + "loss": 0.8534, + "margin_dpo/margin_mean": 71.16682434082031, + "margin_dpo/margin_std": 76.21000671386719, + "step": 222 + }, + { + "KL/chosen_KL_mean": -80.13224029541016, + "KL/mean": -110.2012939453125, + "KL/rejected_KL_mean": -140.27035522460938, + "KL/std": 68.09481811523438, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.012777479365468025, + "fcm_dpo/delta": -0.37129518389701843, + "fcm_dpo/margin": 60.13810348510742, + "fcm_dpo/q_t": 0.3488004803657532, + "grad_norm": 41.97306823730469, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.38986480236053467, + "logits/rejected": -0.36567890644073486, + "logps/chosen": -132.77281188964844, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -223.09536743164062, + "loss": 0.9955, + "margin_dpo/margin_mean": 60.13810729980469, + "margin_dpo/margin_std": 84.43518829345703, + "step": 223 + }, + { + "KL/chosen_KL_mean": -70.65859985351562, + "KL/mean": -108.72351837158203, + "KL/rejected_KL_mean": -146.78843688964844, + "KL/std": 80.6158447265625, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.012629317119717598, + "fcm_dpo/delta": -0.5680704116821289, + "fcm_dpo/margin": 76.12983703613281, + "fcm_dpo/q_t": 0.3225302994251251, + "grad_norm": 41.23021697998047, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.38032078742980957, + "logits/rejected": -0.3519408702850342, + "logps/chosen": -119.25401306152344, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -223.90492248535156, + "loss": 0.8761, + "margin_dpo/margin_mean": 76.12983703613281, + "margin_dpo/margin_std": 88.51429748535156, + "step": 224 + }, + { + "KL/chosen_KL_mean": -80.96829223632812, + "KL/mean": -127.35218048095703, + "KL/rejected_KL_mean": -173.736083984375, + "KL/std": 83.92780303955078, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.012464599683880806, + "fcm_dpo/delta": -0.7653127908706665, + "fcm_dpo/margin": 92.76777648925781, + "fcm_dpo/q_t": 0.2891019582748413, + "grad_norm": 37.687931060791016, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.376278281211853, + "logits/rejected": -0.3558807969093323, + "logps/chosen": -138.96875, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -273.63897705078125, + "loss": 0.7718, + "margin_dpo/margin_mean": 92.76778411865234, + "margin_dpo/margin_std": 94.48170471191406, + "step": 225 + }, + { + "KL/chosen_KL_mean": -70.63302612304688, + "KL/mean": -105.77034759521484, + "KL/rejected_KL_mean": -140.9076690673828, + "KL/std": 76.34187316894531, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.01231297105550766, + "fcm_dpo/delta": -0.47053635120391846, + "fcm_dpo/margin": 70.274658203125, + "fcm_dpo/q_t": 0.3364666998386383, + "grad_norm": 36.470741271972656, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.40814924240112305, + "logits/rejected": -0.35841745138168335, + "logps/chosen": -129.53182983398438, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -219.59542846679688, + "loss": 0.9026, + "margin_dpo/margin_mean": 70.274658203125, + "margin_dpo/margin_std": 82.64395141601562, + "step": 226 + }, + { + "KL/chosen_KL_mean": -83.27752685546875, + "KL/mean": -128.42849731445312, + "KL/rejected_KL_mean": -173.57945251464844, + "KL/std": 92.78634643554688, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.012162862345576286, + "fcm_dpo/delta": -0.7068301439285278, + "fcm_dpo/margin": 90.30192565917969, + "fcm_dpo/q_t": 0.2933015525341034, + "grad_norm": 36.17222213745117, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.3848811686038971, + "logits/rejected": -0.36724066734313965, + "logps/chosen": -142.34970092773438, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -272.9918212890625, + "loss": 0.7985, + "margin_dpo/margin_mean": 90.30192565917969, + "margin_dpo/margin_std": 93.81426239013672, + "step": 227 + }, + { + "KL/chosen_KL_mean": -77.7352523803711, + "KL/mean": -110.19097900390625, + "KL/rejected_KL_mean": -142.64669799804688, + "KL/std": 70.4300537109375, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.012063276022672653, + "fcm_dpo/delta": -0.3859793543815613, + "fcm_dpo/margin": 64.91146087646484, + "fcm_dpo/q_t": 0.3413770794868469, + "grad_norm": 32.25391387939453, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.39154326915740967, + "logits/rejected": -0.3697740435600281, + "logps/chosen": -143.6265411376953, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -233.69546508789062, + "loss": 0.9228, + "margin_dpo/margin_mean": 64.91146087646484, + "margin_dpo/margin_std": 76.83340454101562, + "step": 228 + }, + { + "KL/chosen_KL_mean": -78.18495178222656, + "KL/mean": -115.87548065185547, + "KL/rejected_KL_mean": -153.56600952148438, + "KL/std": 83.92201232910156, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.011948077008128166, + "fcm_dpo/delta": -0.5051275491714478, + "fcm_dpo/margin": 75.38105010986328, + "fcm_dpo/q_t": 0.32153165340423584, + "grad_norm": 42.81840133666992, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.4631323218345642, + "logits/rejected": -0.43050235509872437, + "logps/chosen": -148.89132690429688, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -238.09341430664062, + "loss": 0.8769, + "margin_dpo/margin_mean": 75.38105010986328, + "margin_dpo/margin_std": 85.03656005859375, + "step": 229 + }, + { + "KL/chosen_KL_mean": -66.7181396484375, + "KL/mean": -113.59237670898438, + "KL/rejected_KL_mean": -160.4666290283203, + "KL/std": 83.93539428710938, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.011783900670707226, + "fcm_dpo/delta": -0.7129590511322021, + "fcm_dpo/margin": 93.74847412109375, + "fcm_dpo/q_t": 0.291315495967865, + "grad_norm": 37.50556945800781, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.35473453998565674, + "logits/rejected": -0.35454368591308594, + "logps/chosen": -106.0001449584961, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -246.08853149414062, + "loss": 0.7771, + "margin_dpo/margin_mean": 93.74847412109375, + "margin_dpo/margin_std": 93.64588928222656, + "step": 230 + }, + { + "KL/chosen_KL_mean": -65.77925872802734, + "KL/mean": -101.11721801757812, + "KL/rejected_KL_mean": -136.45516967773438, + "KL/std": 71.96308898925781, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.011669064871966839, + "fcm_dpo/delta": -0.42836901545524597, + "fcm_dpo/margin": 70.67591857910156, + "fcm_dpo/q_t": 0.33771055936813354, + "grad_norm": 39.366085052490234, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.42005205154418945, + "logits/rejected": -0.3887820839881897, + "logps/chosen": -129.05569458007812, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -210.57907104492188, + "loss": 0.8791, + "margin_dpo/margin_mean": 70.67591857910156, + "margin_dpo/margin_std": 80.20990753173828, + "step": 231 + }, + { + "KL/chosen_KL_mean": -80.49052429199219, + "KL/mean": -108.66563415527344, + "KL/rejected_KL_mean": -136.8407440185547, + "KL/std": 78.12565612792969, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.011593952775001526, + "fcm_dpo/delta": -0.25527647137641907, + "fcm_dpo/margin": 56.350215911865234, + "fcm_dpo/q_t": 0.373360276222229, + "grad_norm": 32.06214141845703, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.4245719015598297, + "logits/rejected": -0.39181607961654663, + "logps/chosen": -151.23928833007812, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -220.81781005859375, + "loss": 1.0133, + "margin_dpo/margin_mean": 56.35021209716797, + "margin_dpo/margin_std": 81.52789306640625, + "step": 232 + }, + { + "KL/chosen_KL_mean": -69.39828491210938, + "KL/mean": -112.26213073730469, + "KL/rejected_KL_mean": -155.12594604492188, + "KL/std": 86.15726470947266, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.01148737221956253, + "fcm_dpo/delta": -0.5905115604400635, + "fcm_dpo/margin": 85.72767639160156, + "fcm_dpo/q_t": 0.3252595067024231, + "grad_norm": 34.91195297241211, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.4537864029407501, + "logits/rejected": -0.4480927586555481, + "logps/chosen": -124.28121948242188, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -262.60601806640625, + "loss": 0.8957, + "margin_dpo/margin_mean": 85.72767639160156, + "margin_dpo/margin_std": 106.74943542480469, + "step": 233 + }, + { + "KL/chosen_KL_mean": -66.0254135131836, + "KL/mean": -110.5049819946289, + "KL/rejected_KL_mean": -154.9845428466797, + "KL/std": 75.7353286743164, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.011332383379340172, + "fcm_dpo/delta": -0.6153805255889893, + "fcm_dpo/margin": 88.95913696289062, + "fcm_dpo/q_t": 0.2982660233974457, + "grad_norm": 31.70069122314453, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.3785674571990967, + "logits/rejected": -0.40429338812828064, + "logps/chosen": -110.11993408203125, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -254.99118041992188, + "loss": 0.7774, + "margin_dpo/margin_mean": 88.95913696289062, + "margin_dpo/margin_std": 79.20895385742188, + "step": 234 + }, + { + "KL/chosen_KL_mean": -83.76567077636719, + "KL/mean": -114.40231323242188, + "KL/rejected_KL_mean": -145.03895568847656, + "KL/std": 74.88166809082031, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.011266481131315231, + "fcm_dpo/delta": -0.2922676205635071, + "fcm_dpo/margin": 61.2733039855957, + "fcm_dpo/q_t": 0.36242252588272095, + "grad_norm": 37.513797760009766, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.42455002665519714, + "logits/rejected": -0.39945173263549805, + "logps/chosen": -146.00357055664062, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -235.43402099609375, + "loss": 0.99, + "margin_dpo/margin_mean": 61.27330780029297, + "margin_dpo/margin_std": 83.75914001464844, + "step": 235 + }, + { + "KL/chosen_KL_mean": -62.91929626464844, + "KL/mean": -106.2624740600586, + "KL/rejected_KL_mean": -149.60565185546875, + "KL/std": 80.16526794433594, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.011149470694363117, + "fcm_dpo/delta": -0.5720397233963013, + "fcm_dpo/margin": 86.68635559082031, + "fcm_dpo/q_t": 0.3061680495738983, + "grad_norm": 40.1534309387207, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.38334089517593384, + "logits/rejected": -0.3867339491844177, + "logps/chosen": -112.26066589355469, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -253.11727905273438, + "loss": 0.7867, + "margin_dpo/margin_mean": 86.68635559082031, + "margin_dpo/margin_std": 78.98707580566406, + "step": 236 + }, + { + "KL/chosen_KL_mean": -80.21515655517578, + "KL/mean": -116.89851379394531, + "KL/rejected_KL_mean": -153.5818634033203, + "KL/std": 76.5919189453125, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.011047525331377983, + "fcm_dpo/delta": -0.41392573714256287, + "fcm_dpo/margin": 73.36671447753906, + "fcm_dpo/q_t": 0.3366905748844147, + "grad_norm": 37.10795211791992, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.39750874042510986, + "logits/rejected": -0.3913678228855133, + "logps/chosen": -134.38327026367188, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -248.36224365234375, + "loss": 0.9042, + "margin_dpo/margin_mean": 73.36671447753906, + "margin_dpo/margin_std": 84.40182495117188, + "step": 237 + }, + { + "KL/chosen_KL_mean": -77.66130065917969, + "KL/mean": -112.37471771240234, + "KL/rejected_KL_mean": -147.088134765625, + "KL/std": 76.71656799316406, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.010957850143313408, + "fcm_dpo/delta": -0.3639104962348938, + "fcm_dpo/margin": 69.42684173583984, + "fcm_dpo/q_t": 0.34933486580848694, + "grad_norm": 33.03904724121094, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.35362982749938965, + "logits/rejected": -0.36922675371170044, + "logps/chosen": -131.63442993164062, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -236.506103515625, + "loss": 0.923, + "margin_dpo/margin_mean": 69.42684173583984, + "margin_dpo/margin_std": 83.65792083740234, + "step": 238 + }, + { + "KL/chosen_KL_mean": -84.07707214355469, + "KL/mean": -123.73764038085938, + "KL/rejected_KL_mean": -163.398193359375, + "KL/std": 76.84465026855469, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.010856347158551216, + "fcm_dpo/delta": -0.4660245180130005, + "fcm_dpo/margin": 79.32112884521484, + "fcm_dpo/q_t": 0.3263399004936218, + "grad_norm": 35.94112014770508, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.36835378408432007, + "logits/rejected": -0.3445568084716797, + "logps/chosen": -142.17489624023438, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -256.99114990234375, + "loss": 0.8582, + "margin_dpo/margin_mean": 79.32113647460938, + "margin_dpo/margin_std": 78.72666931152344, + "step": 239 + }, + { + "KL/chosen_KL_mean": -87.8536605834961, + "KL/mean": -125.08583068847656, + "KL/rejected_KL_mean": -162.31800842285156, + "KL/std": 83.88723754882812, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.01077212207019329, + "fcm_dpo/delta": -0.4057100713253021, + "fcm_dpo/margin": 74.46434783935547, + "fcm_dpo/q_t": 0.3358997106552124, + "grad_norm": 37.884498596191406, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.3607942461967468, + "logits/rejected": -0.3369818329811096, + "logps/chosen": -148.4681396484375, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -236.43653869628906, + "loss": 0.9012, + "margin_dpo/margin_mean": 74.46434783935547, + "margin_dpo/margin_std": 81.89598083496094, + "step": 240 + }, + { + "KL/chosen_KL_mean": -78.34124755859375, + "KL/mean": -130.38690185546875, + "KL/rejected_KL_mean": -182.43258666992188, + "KL/std": 94.52236938476562, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.010646835900843143, + "fcm_dpo/delta": -0.7159023284912109, + "fcm_dpo/margin": 104.09135437011719, + "fcm_dpo/q_t": 0.2938423752784729, + "grad_norm": 34.41936111450195, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.38377314805984497, + "logits/rejected": -0.35317492485046387, + "logps/chosen": -144.43231201171875, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -270.49346923828125, + "loss": 0.766, + "margin_dpo/margin_mean": 104.09135437011719, + "margin_dpo/margin_std": 103.39970397949219, + "step": 241 + }, + { + "KL/chosen_KL_mean": -95.9861831665039, + "KL/mean": -130.7741241455078, + "KL/rejected_KL_mean": -165.56207275390625, + "KL/std": 80.13993835449219, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.010557401925325394, + "fcm_dpo/delta": -0.336881160736084, + "fcm_dpo/margin": 69.57588195800781, + "fcm_dpo/q_t": 0.3534601628780365, + "grad_norm": 51.24370193481445, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.30567657947540283, + "logits/rejected": -0.2810847759246826, + "logps/chosen": -163.85011291503906, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -248.92239379882812, + "loss": 0.9388, + "margin_dpo/margin_mean": 69.57588195800781, + "margin_dpo/margin_std": 86.10990905761719, + "step": 242 + }, + { + "KL/chosen_KL_mean": -98.59384155273438, + "KL/mean": -146.91171264648438, + "KL/rejected_KL_mean": -195.22959899902344, + "KL/std": 91.27035522460938, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.010434875264763832, + "fcm_dpo/delta": -0.6146558523178101, + "fcm_dpo/margin": 96.6357421875, + "fcm_dpo/q_t": 0.3010485768318176, + "grad_norm": 34.61061096191406, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.3048647344112396, + "logits/rejected": -0.26071450114250183, + "logps/chosen": -161.67808532714844, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -271.56524658203125, + "loss": 0.7778, + "margin_dpo/margin_mean": 96.6357421875, + "margin_dpo/margin_std": 90.9561996459961, + "step": 243 + }, + { + "KL/chosen_KL_mean": -85.48654174804688, + "KL/mean": -131.46646118164062, + "KL/rejected_KL_mean": -177.44638061523438, + "KL/std": 88.01823425292969, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.010317115113139153, + "fcm_dpo/delta": -0.5540634989738464, + "fcm_dpo/margin": 91.95984649658203, + "fcm_dpo/q_t": 0.3080856204032898, + "grad_norm": 35.35686111450195, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.37389492988586426, + "logits/rejected": -0.36325186491012573, + "logps/chosen": -146.62722778320312, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -272.33831787109375, + "loss": 0.7964, + "margin_dpo/margin_mean": 91.95984649658203, + "margin_dpo/margin_std": 83.81416320800781, + "step": 244 + }, + { + "KL/chosen_KL_mean": -97.7567138671875, + "KL/mean": -140.64154052734375, + "KL/rejected_KL_mean": -183.52639770507812, + "KL/std": 95.3012924194336, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0101974718272686, + "fcm_dpo/delta": -0.4802960157394409, + "fcm_dpo/margin": 85.76968383789062, + "fcm_dpo/q_t": 0.3393905758857727, + "grad_norm": 43.84172821044922, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.3921372890472412, + "logits/rejected": -0.3545539677143097, + "logps/chosen": -165.01901245117188, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -271.16650390625, + "loss": 0.9225, + "margin_dpo/margin_mean": 85.7696762084961, + "margin_dpo/margin_std": 106.13780212402344, + "step": 245 + }, + { + "KL/chosen_KL_mean": -91.89935302734375, + "KL/mean": -132.22682189941406, + "KL/rejected_KL_mean": -172.55429077148438, + "KL/std": 96.4206771850586, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.010124475695192814, + "fcm_dpo/delta": -0.4201340675354004, + "fcm_dpo/margin": 80.6549301147461, + "fcm_dpo/q_t": 0.34418249130249023, + "grad_norm": 35.30609893798828, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.4040786027908325, + "logits/rejected": -0.3637222647666931, + "logps/chosen": -158.5963134765625, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -256.900634765625, + "loss": 0.9238, + "margin_dpo/margin_mean": 80.6549301147461, + "margin_dpo/margin_std": 102.83841705322266, + "step": 246 + }, + { + "KL/chosen_KL_mean": -107.27700805664062, + "KL/mean": -165.2947998046875, + "KL/rejected_KL_mean": -223.31259155273438, + "KL/std": 108.03623962402344, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.009984820149838924, + "fcm_dpo/delta": -0.7679529190063477, + "fcm_dpo/margin": 116.03556823730469, + "fcm_dpo/q_t": 0.2880018353462219, + "grad_norm": 37.433834075927734, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.31970593333244324, + "logits/rejected": -0.3153187334537506, + "logps/chosen": -163.88235473632812, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -329.6058349609375, + "loss": 0.7725, + "margin_dpo/margin_mean": 116.03556823730469, + "margin_dpo/margin_std": 116.6460189819336, + "step": 247 + }, + { + "KL/chosen_KL_mean": -86.13948059082031, + "KL/mean": -130.8503875732422, + "KL/rejected_KL_mean": -175.561279296875, + "KL/std": 79.31680297851562, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.009876626543700695, + "fcm_dpo/delta": -0.48763108253479004, + "fcm_dpo/margin": 89.42179870605469, + "fcm_dpo/q_t": 0.31348514556884766, + "grad_norm": 44.61335372924805, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.3413383364677429, + "logits/rejected": -0.35582929849624634, + "logps/chosen": -130.18270874023438, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -267.41815185546875, + "loss": 0.7971, + "margin_dpo/margin_mean": 89.42180633544922, + "margin_dpo/margin_std": 74.27825164794922, + "step": 248 + }, + { + "KL/chosen_KL_mean": -104.91477966308594, + "KL/mean": -137.20269775390625, + "KL/rejected_KL_mean": -169.4906463623047, + "KL/std": 86.44741821289062, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.009816132485866547, + "fcm_dpo/delta": -0.23551268875598907, + "fcm_dpo/margin": 64.57585144042969, + "fcm_dpo/q_t": 0.37434422969818115, + "grad_norm": 48.107574462890625, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.3671905994415283, + "logits/rejected": -0.34094154834747314, + "logps/chosen": -167.3571319580078, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -249.95870971679688, + "loss": 1.046, + "margin_dpo/margin_mean": 64.57585144042969, + "margin_dpo/margin_std": 99.41860961914062, + "step": 249 + }, + { + "KL/chosen_KL_mean": -90.77139282226562, + "KL/mean": -137.16629028320312, + "KL/rejected_KL_mean": -183.56118774414062, + "KL/std": 79.9173583984375, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.009740087203681469, + "fcm_dpo/delta": -0.5081461668014526, + "fcm_dpo/margin": 92.789794921875, + "fcm_dpo/q_t": 0.3086366355419159, + "grad_norm": 38.40888214111328, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.3681102991104126, + "logits/rejected": -0.30724820494651794, + "logps/chosen": -156.4080810546875, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -257.43304443359375, + "loss": 0.7929, + "margin_dpo/margin_mean": 92.789794921875, + "margin_dpo/margin_std": 75.38710021972656, + "step": 250 + }, + { + "KL/chosen_KL_mean": -109.19084167480469, + "KL/mean": -148.184814453125, + "KL/rejected_KL_mean": -187.17880249023438, + "KL/std": 92.05799865722656, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.009651657193899155, + "fcm_dpo/delta": -0.3557027578353882, + "fcm_dpo/margin": 77.98794555664062, + "fcm_dpo/q_t": 0.34880006313323975, + "grad_norm": 60.541175842285156, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.31971967220306396, + "logits/rejected": -0.28569266200065613, + "logps/chosen": -166.37356567382812, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -264.84222412109375, + "loss": 0.9502, + "margin_dpo/margin_mean": 77.98795318603516, + "margin_dpo/margin_std": 96.10851287841797, + "step": 251 + }, + { + "KL/chosen_KL_mean": -106.32494354248047, + "KL/mean": -154.49526977539062, + "KL/rejected_KL_mean": -202.6655731201172, + "KL/std": 85.48998260498047, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.009559357538819313, + "fcm_dpo/delta": -0.5261793732643127, + "fcm_dpo/margin": 96.34063720703125, + "fcm_dpo/q_t": 0.3016849756240845, + "grad_norm": 36.0700569152832, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.38196754455566406, + "logits/rejected": -0.3372262716293335, + "logps/chosen": -178.01058959960938, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -287.423583984375, + "loss": 0.7973, + "margin_dpo/margin_mean": 96.34063720703125, + "margin_dpo/margin_std": 82.4754638671875, + "step": 252 + }, + { + "KL/chosen_KL_mean": -100.04092407226562, + "KL/mean": -143.52198791503906, + "KL/rejected_KL_mean": -187.00306701660156, + "KL/std": 92.9036865234375, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.009487945586442947, + "fcm_dpo/delta": -0.4284020960330963, + "fcm_dpo/margin": 86.96212768554688, + "fcm_dpo/q_t": 0.33374664187431335, + "grad_norm": 37.04844665527344, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.3867964744567871, + "logits/rejected": -0.37679076194763184, + "logps/chosen": -169.17486572265625, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -285.7055969238281, + "loss": 0.9125, + "margin_dpo/margin_mean": 86.9621353149414, + "margin_dpo/margin_std": 100.89354705810547, + "step": 253 + }, + { + "KL/chosen_KL_mean": -111.02886962890625, + "KL/mean": -151.83905029296875, + "KL/rejected_KL_mean": -192.64923095703125, + "KL/std": 97.16827392578125, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.00940469466149807, + "fcm_dpo/delta": -0.3705146908760071, + "fcm_dpo/margin": 81.62034606933594, + "fcm_dpo/q_t": 0.35719332098960876, + "grad_norm": 40.747154235839844, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.35018646717071533, + "logits/rejected": -0.32813340425491333, + "logps/chosen": -165.18386840820312, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -282.95684814453125, + "loss": 0.9848, + "margin_dpo/margin_mean": 81.62034606933594, + "margin_dpo/margin_std": 116.91175842285156, + "step": 254 + }, + { + "KL/chosen_KL_mean": -116.6124267578125, + "KL/mean": -160.88131713867188, + "KL/rejected_KL_mean": -205.15020751953125, + "KL/std": 94.34854125976562, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.009320816025137901, + "fcm_dpo/delta": -0.42926257848739624, + "fcm_dpo/margin": 88.53778076171875, + "fcm_dpo/q_t": 0.342517614364624, + "grad_norm": 49.83819580078125, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.329836368560791, + "logits/rejected": -0.32322707772254944, + "logps/chosen": -173.75408935546875, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -295.3587951660156, + "loss": 0.964, + "margin_dpo/margin_mean": 88.53778076171875, + "margin_dpo/margin_std": 117.14259338378906, + "step": 255 + }, + { + "KL/chosen_KL_mean": -101.06261444091797, + "KL/mean": -150.05838012695312, + "KL/rejected_KL_mean": -199.05410766601562, + "KL/std": 102.71546173095703, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.0092360470443964, + "fcm_dpo/delta": -0.5097526907920837, + "fcm_dpo/margin": 97.99149322509766, + "fcm_dpo/q_t": 0.32233816385269165, + "grad_norm": 45.70084762573242, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.25965771079063416, + "logits/rejected": -0.2544403076171875, + "logps/chosen": -156.22610473632812, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -291.6170349121094, + "loss": 0.8567, + "margin_dpo/margin_mean": 97.99149322509766, + "margin_dpo/margin_std": 105.54132080078125, + "step": 256 + }, + { + "KL/chosen_KL_mean": -94.38179779052734, + "KL/mean": -144.68923950195312, + "KL/rejected_KL_mean": -194.9966583251953, + "KL/std": 101.10047912597656, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.009128324687480927, + "fcm_dpo/delta": -0.5244594216346741, + "fcm_dpo/margin": 100.61486053466797, + "fcm_dpo/q_t": 0.324202299118042, + "grad_norm": 33.801448822021484, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.32540446519851685, + "logits/rejected": -0.3128937780857086, + "logps/chosen": -143.8054962158203, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -274.5345764160156, + "loss": 0.8622, + "margin_dpo/margin_mean": 100.61486053466797, + "margin_dpo/margin_std": 107.0750503540039, + "step": 257 + }, + { + "KL/chosen_KL_mean": -108.99232482910156, + "KL/mean": -152.51171875, + "KL/rejected_KL_mean": -196.03115844726562, + "KL/std": 93.95271301269531, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.009066222235560417, + "fcm_dpo/delta": -0.3921354413032532, + "fcm_dpo/margin": 87.03881072998047, + "fcm_dpo/q_t": 0.33922919631004333, + "grad_norm": 38.24882507324219, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.2920841574668884, + "logits/rejected": -0.2849876880645752, + "logps/chosen": -168.37644958496094, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -292.021240234375, + "loss": 0.9491, + "margin_dpo/margin_mean": 87.03880310058594, + "margin_dpo/margin_std": 110.92609405517578, + "step": 258 + }, + { + "KL/chosen_KL_mean": -93.82998657226562, + "KL/mean": -133.01773071289062, + "KL/rejected_KL_mean": -172.2054901123047, + "KL/std": 86.7385482788086, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.009000067599117756, + "fcm_dpo/delta": -0.3077760338783264, + "fcm_dpo/margin": 78.37548828125, + "fcm_dpo/q_t": 0.3563136160373688, + "grad_norm": 34.161563873291016, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.3999626338481903, + "logits/rejected": -0.3934265673160553, + "logps/chosen": -146.6583251953125, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -261.39715576171875, + "loss": 0.966, + "margin_dpo/margin_mean": 78.37548828125, + "margin_dpo/margin_std": 98.01071166992188, + "step": 259 + }, + { + "KL/chosen_KL_mean": -91.75828552246094, + "KL/mean": -136.31024169921875, + "KL/rejected_KL_mean": -180.86219787597656, + "KL/std": 88.80322265625, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.008940936997532845, + "fcm_dpo/delta": -0.3997294306755066, + "fcm_dpo/margin": 89.10391235351562, + "fcm_dpo/q_t": 0.3381941616535187, + "grad_norm": 38.0841064453125, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.37579652667045593, + "logits/rejected": -0.3916095197200775, + "logps/chosen": -139.17596435546875, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -275.95196533203125, + "loss": 0.8817, + "margin_dpo/margin_mean": 89.10391235351562, + "margin_dpo/margin_std": 95.78402709960938, + "step": 260 + }, + { + "KL/chosen_KL_mean": -81.51264190673828, + "KL/mean": -129.02117919921875, + "KL/rejected_KL_mean": -176.52972412109375, + "KL/std": 90.31359100341797, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.008858158253133297, + "fcm_dpo/delta": -0.44553142786026, + "fcm_dpo/margin": 95.01707458496094, + "fcm_dpo/q_t": 0.32786160707473755, + "grad_norm": 35.34941482543945, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.3139702379703522, + "logits/rejected": -0.30691468715667725, + "logps/chosen": -134.5440216064453, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -265.044677734375, + "loss": 0.8573, + "margin_dpo/margin_mean": 95.01707458496094, + "margin_dpo/margin_std": 97.85682678222656, + "step": 261 + }, + { + "KL/chosen_KL_mean": -95.03675842285156, + "KL/mean": -133.72930908203125, + "KL/rejected_KL_mean": -172.42184448242188, + "KL/std": 79.69241333007812, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.008805808611214161, + "fcm_dpo/delta": -0.28329652547836304, + "fcm_dpo/margin": 77.38510131835938, + "fcm_dpo/q_t": 0.350163996219635, + "grad_norm": 34.43601989746094, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.27383410930633545, + "logits/rejected": -0.24388615787029266, + "logps/chosen": -154.65689086914062, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -258.84039306640625, + "loss": 0.9254, + "margin_dpo/margin_mean": 77.38510131835938, + "margin_dpo/margin_std": 79.50154876708984, + "step": 262 + }, + { + "KL/chosen_KL_mean": -89.35324096679688, + "KL/mean": -143.482421875, + "KL/rejected_KL_mean": -197.61160278320312, + "KL/std": 98.19929504394531, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.008715115487575531, + "fcm_dpo/delta": -0.5488941669464111, + "fcm_dpo/margin": 108.25836181640625, + "fcm_dpo/q_t": 0.3076201379299164, + "grad_norm": 50.25643539428711, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.29875218868255615, + "logits/rejected": -0.26528897881507874, + "logps/chosen": -148.77418518066406, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -294.46881103515625, + "loss": 0.8009, + "margin_dpo/margin_mean": 108.25835418701172, + "margin_dpo/margin_std": 98.67405700683594, + "step": 263 + }, + { + "KL/chosen_KL_mean": -94.08186340332031, + "KL/mean": -138.84451293945312, + "KL/rejected_KL_mean": -183.607177734375, + "KL/std": 87.13362121582031, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.008639801293611526, + "fcm_dpo/delta": -0.37672215700149536, + "fcm_dpo/margin": 89.52530670166016, + "fcm_dpo/q_t": 0.3379225432872772, + "grad_norm": 58.334564208984375, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.3002208173274994, + "logits/rejected": -0.2839544415473938, + "logps/chosen": -156.803955078125, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -277.46337890625, + "loss": 0.8822, + "margin_dpo/margin_mean": 89.52530670166016, + "margin_dpo/margin_std": 90.42294311523438, + "step": 264 + }, + { + "KL/chosen_KL_mean": -112.17031860351562, + "KL/mean": -156.56813049316406, + "KL/rejected_KL_mean": -200.9659423828125, + "KL/std": 100.57118225097656, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.008582094684243202, + "fcm_dpo/delta": -0.36483150720596313, + "fcm_dpo/margin": 88.79563903808594, + "fcm_dpo/q_t": 0.35009145736694336, + "grad_norm": 41.660911560058594, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.21025711297988892, + "logits/rejected": -0.18984851241111755, + "logps/chosen": -174.14178466796875, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -288.9865417480469, + "loss": 0.9367, + "margin_dpo/margin_mean": 88.79563903808594, + "margin_dpo/margin_std": 110.92127990722656, + "step": 265 + }, + { + "KL/chosen_KL_mean": -106.98106384277344, + "KL/mean": -150.916015625, + "KL/rejected_KL_mean": -194.8509521484375, + "KL/std": 96.85904693603516, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.008518719114363194, + "fcm_dpo/delta": -0.3512975573539734, + "fcm_dpo/margin": 87.86988830566406, + "fcm_dpo/q_t": 0.3446722626686096, + "grad_norm": 48.0034065246582, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.2845054864883423, + "logits/rejected": -0.2280675768852234, + "logps/chosen": -174.0807342529297, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -262.8221740722656, + "loss": 0.9122, + "margin_dpo/margin_mean": 87.86988830566406, + "margin_dpo/margin_std": 96.51178741455078, + "step": 266 + }, + { + "KL/chosen_KL_mean": -107.95355987548828, + "KL/mean": -155.80191040039062, + "KL/rejected_KL_mean": -203.6502685546875, + "KL/std": 96.25630187988281, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.008450968191027641, + "fcm_dpo/delta": -0.41223710775375366, + "fcm_dpo/margin": 95.69668579101562, + "fcm_dpo/q_t": 0.32934847474098206, + "grad_norm": 41.43006134033203, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.2696833312511444, + "logits/rejected": -0.23559030890464783, + "logps/chosen": -176.92431640625, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -293.8187255859375, + "loss": 0.8524, + "margin_dpo/margin_mean": 95.6966781616211, + "margin_dpo/margin_std": 91.59297180175781, + "step": 267 + }, + { + "KL/chosen_KL_mean": -121.45596313476562, + "KL/mean": -163.731201171875, + "KL/rejected_KL_mean": -206.00643920898438, + "KL/std": 88.85858154296875, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0083943922072649, + "fcm_dpo/delta": -0.3120897710323334, + "fcm_dpo/margin": 84.55046844482422, + "fcm_dpo/q_t": 0.35003983974456787, + "grad_norm": 63.71186065673828, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.18893195688724518, + "logits/rejected": -0.1957235038280487, + "logps/chosen": -177.3562774658203, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -307.654052734375, + "loss": 0.9303, + "margin_dpo/margin_mean": 84.55047607421875, + "margin_dpo/margin_std": 96.22168731689453, + "step": 268 + }, + { + "KL/chosen_KL_mean": -135.60183715820312, + "KL/mean": -194.59112548828125, + "KL/rejected_KL_mean": -253.5803985595703, + "KL/std": 107.18486022949219, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.008308888413012028, + "fcm_dpo/delta": -0.5861600041389465, + "fcm_dpo/margin": 117.97856140136719, + "fcm_dpo/q_t": 0.3095221221446991, + "grad_norm": 73.39482116699219, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.22161518037319183, + "logits/rejected": -0.19611665606498718, + "logps/chosen": -205.6414031982422, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -360.9297790527344, + "loss": 0.8378, + "margin_dpo/margin_mean": 117.97856140136719, + "margin_dpo/margin_std": 124.66133117675781, + "step": 269 + }, + { + "KL/chosen_KL_mean": -124.41390991210938, + "KL/mean": -168.064453125, + "KL/rejected_KL_mean": -211.71499633789062, + "KL/std": 100.48226928710938, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.008248567581176758, + "fcm_dpo/delta": -0.3223602771759033, + "fcm_dpo/margin": 87.30109405517578, + "fcm_dpo/q_t": 0.3568444550037384, + "grad_norm": 45.41849136352539, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.27393507957458496, + "logits/rejected": -0.26879918575286865, + "logps/chosen": -193.9473876953125, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -321.6436462402344, + "loss": 0.9742, + "margin_dpo/margin_mean": 87.30110168457031, + "margin_dpo/margin_std": 116.33839416503906, + "step": 270 + }, + { + "KL/chosen_KL_mean": -120.69148254394531, + "KL/mean": -177.29116821289062, + "KL/rejected_KL_mean": -233.89085388183594, + "KL/std": 99.96987915039062, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.008165929466485977, + "fcm_dpo/delta": -0.5294230580329895, + "fcm_dpo/margin": 113.19938659667969, + "fcm_dpo/q_t": 0.3152883052825928, + "grad_norm": 31.533729553222656, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.22561264038085938, + "logits/rejected": -0.1990375816822052, + "logps/chosen": -177.4560546875, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -326.4046936035156, + "loss": 0.8137, + "margin_dpo/margin_mean": 113.19938659667969, + "margin_dpo/margin_std": 109.6073226928711, + "step": 271 + }, + { + "KL/chosen_KL_mean": -114.31645202636719, + "KL/mean": -178.15872192382812, + "KL/rejected_KL_mean": -242.00100708007812, + "KL/std": 113.5208740234375, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.008074797689914703, + "fcm_dpo/delta": -0.6372488141059875, + "fcm_dpo/margin": 127.6845474243164, + "fcm_dpo/q_t": 0.29747164249420166, + "grad_norm": 42.694454193115234, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.17121776938438416, + "logits/rejected": -0.1862572282552719, + "logps/chosen": -163.8135986328125, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -347.5438232421875, + "loss": 0.7923, + "margin_dpo/margin_mean": 127.68453979492188, + "margin_dpo/margin_std": 122.64485168457031, + "step": 272 + }, + { + "KL/chosen_KL_mean": -127.05171966552734, + "KL/mean": -190.59274291992188, + "KL/rejected_KL_mean": -254.1337432861328, + "KL/std": 112.17765045166016, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.007962321862578392, + "fcm_dpo/delta": -0.6187852621078491, + "fcm_dpo/margin": 127.08203125, + "fcm_dpo/q_t": 0.30050593614578247, + "grad_norm": 34.17970275878906, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.18308264017105103, + "logits/rejected": -0.15806245803833008, + "logps/chosen": -190.02711486816406, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -346.63232421875, + "loss": 0.7785, + "margin_dpo/margin_mean": 127.08203125, + "margin_dpo/margin_std": 116.11636352539062, + "step": 273 + }, + { + "KL/chosen_KL_mean": -151.47463989257812, + "KL/mean": -203.51943969726562, + "KL/rejected_KL_mean": -255.56423950195312, + "KL/std": 103.97351837158203, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.007894854061305523, + "fcm_dpo/delta": -0.42521557211875916, + "fcm_dpo/margin": 104.089599609375, + "fcm_dpo/q_t": 0.3345829248428345, + "grad_norm": 41.87339782714844, + "learning_rate": 3.75e-07, + "logits/chosen": -0.13648435473442078, + "logits/rejected": -0.11335180699825287, + "logps/chosen": -207.14234924316406, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -332.8973388671875, + "loss": 0.8947, + "margin_dpo/margin_mean": 104.089599609375, + "margin_dpo/margin_std": 116.11741638183594, + "step": 274 + }, + { + "KL/chosen_KL_mean": -121.24569702148438, + "KL/mean": -174.28024291992188, + "KL/rejected_KL_mean": -227.3148193359375, + "KL/std": 110.29039764404297, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.007830331102013588, + "fcm_dpo/delta": -0.4339829683303833, + "fcm_dpo/margin": 106.06912231445312, + "fcm_dpo/q_t": 0.33614403009414673, + "grad_norm": 39.99098205566406, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.15557903051376343, + "logits/rejected": -0.15859189629554749, + "logps/chosen": -169.8404083251953, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -320.6185302734375, + "loss": 0.9099, + "margin_dpo/margin_mean": 106.0691146850586, + "margin_dpo/margin_std": 123.91403198242188, + "step": 275 + }, + { + "KL/chosen_KL_mean": -134.39389038085938, + "KL/mean": -193.3289794921875, + "KL/rejected_KL_mean": -252.2640838623047, + "KL/std": 116.9022216796875, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.007745642215013504, + "fcm_dpo/delta": -0.5179329514503479, + "fcm_dpo/margin": 117.87018585205078, + "fcm_dpo/q_t": 0.32436609268188477, + "grad_norm": 38.76327133178711, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.16366349160671234, + "logits/rejected": -0.13802406191825867, + "logps/chosen": -190.9713134765625, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -322.6297607421875, + "loss": 0.8621, + "margin_dpo/margin_mean": 117.87017822265625, + "margin_dpo/margin_std": 132.2515869140625, + "step": 276 + }, + { + "KL/chosen_KL_mean": -151.2074432373047, + "KL/mean": -209.17526245117188, + "KL/rejected_KL_mean": -267.14306640625, + "KL/std": 111.79081726074219, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.007674789987504482, + "fcm_dpo/delta": -0.4940037727355957, + "fcm_dpo/margin": 115.93562316894531, + "fcm_dpo/q_t": 0.32357341051101685, + "grad_norm": 35.18100357055664, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.14009565114974976, + "logits/rejected": -0.12759149074554443, + "logps/chosen": -207.47900390625, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -360.02435302734375, + "loss": 0.8475, + "margin_dpo/margin_mean": 115.93562316894531, + "margin_dpo/margin_std": 119.72349548339844, + "step": 277 + }, + { + "KL/chosen_KL_mean": -142.2149200439453, + "KL/mean": -203.19778442382812, + "KL/rejected_KL_mean": -264.1806640625, + "KL/std": 124.36817932128906, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.007590082008391619, + "fcm_dpo/delta": -0.5307799577713013, + "fcm_dpo/margin": 121.96574401855469, + "fcm_dpo/q_t": 0.3211669921875, + "grad_norm": 59.06837844848633, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.12663665413856506, + "logits/rejected": -0.134543776512146, + "logps/chosen": -195.1568603515625, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -355.43426513671875, + "loss": 0.8476, + "margin_dpo/margin_mean": 121.96574401855469, + "margin_dpo/margin_std": 131.10198974609375, + "step": 278 + }, + { + "KL/chosen_KL_mean": -168.8824462890625, + "KL/mean": -236.0039520263672, + "KL/rejected_KL_mean": -303.12548828125, + "KL/std": 132.05711364746094, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.007494003511965275, + "fcm_dpo/delta": -0.6131243109703064, + "fcm_dpo/margin": 134.2430419921875, + "fcm_dpo/q_t": 0.31640303134918213, + "grad_norm": 56.03714370727539, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.04074610397219658, + "logits/rejected": -0.03914094343781471, + "logps/chosen": -217.5237579345703, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -390.9769287109375, + "loss": 0.8745, + "margin_dpo/margin_mean": 134.2430419921875, + "margin_dpo/margin_std": 157.28538513183594, + "step": 279 + }, + { + "KL/chosen_KL_mean": -152.41552734375, + "KL/mean": -219.79782104492188, + "KL/rejected_KL_mean": -287.18011474609375, + "KL/std": 114.4323959350586, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.007409612648189068, + "fcm_dpo/delta": -0.6048027873039246, + "fcm_dpo/margin": 134.7646026611328, + "fcm_dpo/q_t": 0.29286009073257446, + "grad_norm": 56.92327880859375, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.0756160318851471, + "logits/rejected": -0.07629910111427307, + "logps/chosen": -211.212646484375, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -385.7989807128906, + "loss": 0.7498, + "margin_dpo/margin_mean": 134.7646026611328, + "margin_dpo/margin_std": 107.83329772949219, + "step": 280 + }, + { + "KL/chosen_KL_mean": -133.2591552734375, + "KL/mean": -193.34909057617188, + "KL/rejected_KL_mean": -253.43905639648438, + "KL/std": 111.77217864990234, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.007339444011449814, + "fcm_dpo/delta": -0.48615387082099915, + "fcm_dpo/margin": 120.17987060546875, + "fcm_dpo/q_t": 0.31156760454177856, + "grad_norm": 50.50090026855469, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.16049012541770935, + "logits/rejected": -0.13395386934280396, + "logps/chosen": -188.7476806640625, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -334.3216247558594, + "loss": 0.8273, + "margin_dpo/margin_mean": 120.17987060546875, + "margin_dpo/margin_std": 111.01317596435547, + "step": 281 + }, + { + "KL/chosen_KL_mean": -140.9694061279297, + "KL/mean": -189.86151123046875, + "KL/rejected_KL_mean": -238.75360107421875, + "KL/std": 103.66000366210938, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.007287952117621899, + "fcm_dpo/delta": -0.3147735595703125, + "fcm_dpo/margin": 97.7842025756836, + "fcm_dpo/q_t": 0.3533177375793457, + "grad_norm": 35.283180236816406, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.2301567792892456, + "logits/rejected": -0.19582641124725342, + "logps/chosen": -214.03955078125, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -334.1045837402344, + "loss": 0.9268, + "margin_dpo/margin_mean": 97.7842025756836, + "margin_dpo/margin_std": 114.606689453125, + "step": 282 + }, + { + "KL/chosen_KL_mean": -146.08079528808594, + "KL/mean": -212.4748992919922, + "KL/rejected_KL_mean": -278.8689880371094, + "KL/std": 134.80052185058594, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.007213708013296127, + "fcm_dpo/delta": -0.5631073713302612, + "fcm_dpo/margin": 132.7882080078125, + "fcm_dpo/q_t": 0.30530333518981934, + "grad_norm": 43.42702865600586, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.131260946393013, + "logits/rejected": -0.12962763011455536, + "logps/chosen": -207.979248046875, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -375.85552978515625, + "loss": 0.801, + "margin_dpo/margin_mean": 132.7882080078125, + "margin_dpo/margin_std": 123.63528442382812, + "step": 283 + }, + { + "KL/chosen_KL_mean": -152.048583984375, + "KL/mean": -219.50747680664062, + "KL/rejected_KL_mean": -286.96636962890625, + "KL/std": 136.01751708984375, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.007129607256501913, + "fcm_dpo/delta": -0.5673815011978149, + "fcm_dpo/margin": 134.91778564453125, + "fcm_dpo/q_t": 0.3086276054382324, + "grad_norm": 36.27334213256836, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.12349693477153778, + "logits/rejected": -0.10875552892684937, + "logps/chosen": -210.48410034179688, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -380.4356384277344, + "loss": 0.789, + "margin_dpo/margin_mean": 134.91778564453125, + "margin_dpo/margin_std": 128.8331298828125, + "step": 284 + }, + { + "KL/chosen_KL_mean": -172.4208526611328, + "KL/mean": -231.86831665039062, + "KL/rejected_KL_mean": -291.3157958984375, + "KL/std": 116.45860290527344, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.007067810744047165, + "fcm_dpo/delta": -0.4438358545303345, + "fcm_dpo/margin": 118.89495086669922, + "fcm_dpo/q_t": 0.32199278473854065, + "grad_norm": 44.77175521850586, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.11267679929733276, + "logits/rejected": -0.09678535908460617, + "logps/chosen": -238.65304565429688, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -390.442626953125, + "loss": 0.8473, + "margin_dpo/margin_mean": 118.89494323730469, + "margin_dpo/margin_std": 111.09343719482422, + "step": 285 + }, + { + "KL/chosen_KL_mean": -176.44415283203125, + "KL/mean": -244.51162719726562, + "KL/rejected_KL_mean": -312.5791015625, + "KL/std": 138.4091033935547, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.006989085115492344, + "fcm_dpo/delta": -0.556684672832489, + "fcm_dpo/margin": 136.13494873046875, + "fcm_dpo/q_t": 0.3105573058128357, + "grad_norm": 49.819984436035156, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.11334480345249176, + "logits/rejected": -0.0909290462732315, + "logps/chosen": -249.39517211914062, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -401.16754150390625, + "loss": 0.8202, + "margin_dpo/margin_mean": 136.13494873046875, + "margin_dpo/margin_std": 135.00865173339844, + "step": 286 + }, + { + "KL/chosen_KL_mean": -181.22718811035156, + "KL/mean": -237.77279663085938, + "KL/rejected_KL_mean": -294.3183898925781, + "KL/std": 121.50303649902344, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.006931045092642307, + "fcm_dpo/delta": -0.3867529630661011, + "fcm_dpo/margin": 113.09121704101562, + "fcm_dpo/q_t": 0.34309762716293335, + "grad_norm": 49.853336334228516, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": 0.039054177701473236, + "logits/rejected": 0.06069245561957359, + "logps/chosen": -242.76834106445312, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -372.01446533203125, + "loss": 0.9204, + "margin_dpo/margin_mean": 113.0912094116211, + "margin_dpo/margin_std": 135.43905639648438, + "step": 287 + }, + { + "KL/chosen_KL_mean": -178.58743286132812, + "KL/mean": -243.94699096679688, + "KL/rejected_KL_mean": -309.3065185546875, + "KL/std": 117.5995864868164, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.00685396883636713, + "fcm_dpo/delta": -0.5014618039131165, + "fcm_dpo/margin": 130.71908569335938, + "fcm_dpo/q_t": 0.3135807514190674, + "grad_norm": 46.38627624511719, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.04959460347890854, + "logits/rejected": -0.032548222690820694, + "logps/chosen": -235.2486572265625, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -396.6422119140625, + "loss": 0.8223, + "margin_dpo/margin_mean": 130.71908569335938, + "margin_dpo/margin_std": 116.14686584472656, + "step": 288 + }, + { + "KL/chosen_KL_mean": -170.56729125976562, + "KL/mean": -244.06292724609375, + "KL/rejected_KL_mean": -317.55859375, + "KL/std": 135.35475158691406, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.006787968799471855, + "fcm_dpo/delta": -0.6035319566726685, + "fcm_dpo/margin": 146.9912872314453, + "fcm_dpo/q_t": 0.30728477239608765, + "grad_norm": 39.913211822509766, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": 0.049535252153873444, + "logits/rejected": 0.05334234982728958, + "logps/chosen": -215.79766845703125, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -405.20123291015625, + "loss": 0.8031, + "margin_dpo/margin_mean": 146.9912872314453, + "margin_dpo/margin_std": 144.833251953125, + "step": 289 + }, + { + "KL/chosen_KL_mean": -179.87001037597656, + "KL/mean": -252.52294921875, + "KL/rejected_KL_mean": -325.17584228515625, + "KL/std": 141.1082763671875, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.006704019848257303, + "fcm_dpo/delta": -0.5798993110656738, + "fcm_dpo/margin": 145.30584716796875, + "fcm_dpo/q_t": 0.31299489736557007, + "grad_norm": 47.96451187133789, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.004686217755079269, + "logits/rejected": -0.03104211576282978, + "logps/chosen": -235.3415069580078, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -441.8844299316406, + "loss": 0.8278, + "margin_dpo/margin_mean": 145.30584716796875, + "margin_dpo/margin_std": 155.413818359375, + "step": 290 + }, + { + "KL/chosen_KL_mean": -150.41099548339844, + "KL/mean": -217.92784118652344, + "KL/rejected_KL_mean": -285.44464111328125, + "KL/std": 119.50800323486328, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.006642194464802742, + "fcm_dpo/delta": -0.5011211633682251, + "fcm_dpo/margin": 135.03366088867188, + "fcm_dpo/q_t": 0.315216064453125, + "grad_norm": 50.70289993286133, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.039820730686187744, + "logits/rejected": -0.02865706942975521, + "logps/chosen": -213.67135620117188, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -374.74176025390625, + "loss": 0.8242, + "margin_dpo/margin_mean": 135.03366088867188, + "margin_dpo/margin_std": 124.2327880859375, + "step": 291 + }, + { + "KL/chosen_KL_mean": -186.73133850097656, + "KL/mean": -253.09695434570312, + "KL/rejected_KL_mean": -319.4625549316406, + "KL/std": 137.81271362304688, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.006575836334377527, + "fcm_dpo/delta": -0.476773202419281, + "fcm_dpo/margin": 132.73123168945312, + "fcm_dpo/q_t": 0.324418842792511, + "grad_norm": 64.69921112060547, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": 0.01930868625640869, + "logits/rejected": 0.0293303020298481, + "logps/chosen": -240.64987182617188, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -409.4239501953125, + "loss": 0.9119, + "margin_dpo/margin_mean": 132.73123168945312, + "margin_dpo/margin_std": 159.21224975585938, + "step": 292 + }, + { + "KL/chosen_KL_mean": -163.05697631835938, + "KL/mean": -220.07232666015625, + "KL/rejected_KL_mean": -277.08770751953125, + "KL/std": 131.9326629638672, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.006525643169879913, + "fcm_dpo/delta": -0.3465798497200012, + "fcm_dpo/margin": 114.0307388305664, + "fcm_dpo/q_t": 0.3527754545211792, + "grad_norm": 41.0057373046875, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": 0.00332890497520566, + "logits/rejected": 0.006756210699677467, + "logps/chosen": -223.43299865722656, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -354.94012451171875, + "loss": 0.9646, + "margin_dpo/margin_mean": 114.0307388305664, + "margin_dpo/margin_std": 146.9138641357422, + "step": 293 + }, + { + "KL/chosen_KL_mean": -147.04562377929688, + "KL/mean": -210.9862823486328, + "KL/rejected_KL_mean": -274.92694091796875, + "KL/std": 135.0050048828125, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.006465350277721882, + "fcm_dpo/delta": -0.43067580461502075, + "fcm_dpo/margin": 127.88129425048828, + "fcm_dpo/q_t": 0.3395842909812927, + "grad_norm": 36.16399002075195, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.015080882236361504, + "logits/rejected": -0.017156781628727913, + "logps/chosen": -195.1331787109375, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -356.82391357421875, + "loss": 0.9178, + "margin_dpo/margin_mean": 127.88130187988281, + "margin_dpo/margin_std": 156.6295166015625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -170.8695526123047, + "KL/mean": -233.734375, + "KL/rejected_KL_mean": -296.5992431640625, + "KL/std": 129.63088989257812, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.006414244882762432, + "fcm_dpo/delta": -0.40986472368240356, + "fcm_dpo/margin": 125.72966003417969, + "fcm_dpo/q_t": 0.3361758589744568, + "grad_norm": 44.23348617553711, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.01203520130366087, + "logits/rejected": -0.013470523059368134, + "logps/chosen": -220.79421997070312, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -384.0555419921875, + "loss": 0.8965, + "margin_dpo/margin_mean": 125.72966003417969, + "margin_dpo/margin_std": 139.5579833984375, + "step": 295 + }, + { + "KL/chosen_KL_mean": -188.01974487304688, + "KL/mean": -237.75152587890625, + "KL/rejected_KL_mean": -287.48333740234375, + "KL/std": 122.777587890625, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.00637521967291832, + "fcm_dpo/delta": -0.23587414622306824, + "fcm_dpo/margin": 99.46358489990234, + "fcm_dpo/q_t": 0.36916521191596985, + "grad_norm": 65.02623748779297, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.07346773147583008, + "logits/rejected": -0.05706843361258507, + "logps/chosen": -253.51097106933594, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -380.5723876953125, + "loss": 1.0265, + "margin_dpo/margin_mean": 99.46358489990234, + "margin_dpo/margin_std": 141.9931640625, + "step": 296 + }, + { + "KL/chosen_KL_mean": -169.35629272460938, + "KL/mean": -226.46038818359375, + "KL/rejected_KL_mean": -283.56451416015625, + "KL/std": 120.45196533203125, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.006340652704238892, + "fcm_dpo/delta": -0.32655054330825806, + "fcm_dpo/margin": 114.208251953125, + "fcm_dpo/q_t": 0.34385746717453003, + "grad_norm": 42.42692184448242, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.10414460301399231, + "logits/rejected": -0.11292436718940735, + "logps/chosen": -225.83322143554688, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -378.70306396484375, + "loss": 0.8941, + "margin_dpo/margin_mean": 114.208251953125, + "margin_dpo/margin_std": 111.82743835449219, + "step": 297 + }, + { + "KL/chosen_KL_mean": -170.97262573242188, + "KL/mean": -237.14398193359375, + "KL/rejected_KL_mean": -303.31536865234375, + "KL/std": 142.14178466796875, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.006290923338383436, + "fcm_dpo/delta": -0.43610846996307373, + "fcm_dpo/margin": 132.34274291992188, + "fcm_dpo/q_t": 0.3363783359527588, + "grad_norm": 42.390769958496094, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.06253157556056976, + "logits/rejected": -0.08549435436725616, + "logps/chosen": -238.29779052734375, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -419.9775390625, + "loss": 0.8839, + "margin_dpo/margin_mean": 132.3427276611328, + "margin_dpo/margin_std": 150.4270782470703, + "step": 298 + }, + { + "KL/chosen_KL_mean": -155.95704650878906, + "KL/mean": -222.50294494628906, + "KL/rejected_KL_mean": -289.048828125, + "KL/std": 130.13038635253906, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.006241428665816784, + "fcm_dpo/delta": -0.43405744433403015, + "fcm_dpo/margin": 133.091796875, + "fcm_dpo/q_t": 0.3366101086139679, + "grad_norm": 49.890594482421875, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.07402372360229492, + "logits/rejected": -0.07371948659420013, + "logps/chosen": -204.9191436767578, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -373.3770751953125, + "loss": 0.9007, + "margin_dpo/margin_mean": 133.091796875, + "margin_dpo/margin_std": 151.79476928710938, + "step": 299 + }, + { + "KL/chosen_KL_mean": -211.30166625976562, + "KL/mean": -285.75286865234375, + "KL/rejected_KL_mean": -360.2040100097656, + "KL/std": 152.29135131835938, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.006175283342599869, + "fcm_dpo/delta": -0.5241683721542358, + "fcm_dpo/margin": 148.90234375, + "fcm_dpo/q_t": 0.32155176997184753, + "grad_norm": 55.54924774169922, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.018721885979175568, + "logits/rejected": -0.013455286622047424, + "logps/chosen": -270.3753662109375, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -456.17047119140625, + "loss": 0.8919, + "margin_dpo/margin_mean": 148.90234375, + "margin_dpo/margin_std": 173.66775512695312, + "step": 300 + }, + { + "KL/chosen_KL_mean": -193.0668182373047, + "KL/mean": -266.8564453125, + "KL/rejected_KL_mean": -340.64605712890625, + "KL/std": 138.2484588623047, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.00610553240403533, + "fcm_dpo/delta": -0.5061157941818237, + "fcm_dpo/margin": 147.57923889160156, + "fcm_dpo/q_t": 0.32142043113708496, + "grad_norm": 40.89961242675781, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.03288700804114342, + "logits/rejected": -0.03299275413155556, + "logps/chosen": -250.31619262695312, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -432.9996032714844, + "loss": 0.8345, + "margin_dpo/margin_mean": 147.57923889160156, + "margin_dpo/margin_std": 148.95220947265625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -166.41244506835938, + "KL/mean": -231.26895141601562, + "KL/rejected_KL_mean": -296.1254577636719, + "KL/std": 127.39794158935547, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.006055896170437336, + "fcm_dpo/delta": -0.38886570930480957, + "fcm_dpo/margin": 129.7130126953125, + "fcm_dpo/q_t": 0.339442640542984, + "grad_norm": 55.346649169921875, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.06920363008975983, + "logits/rejected": -0.06983371078968048, + "logps/chosen": -217.61044311523438, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -393.351806640625, + "loss": 0.8976, + "margin_dpo/margin_mean": 129.7130126953125, + "margin_dpo/margin_std": 143.9541778564453, + "step": 302 + }, + { + "KL/chosen_KL_mean": -188.43368530273438, + "KL/mean": -248.5778045654297, + "KL/rejected_KL_mean": -308.721923828125, + "KL/std": 137.1468048095703, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.0060157570987939835, + "fcm_dpo/delta": -0.32613593339920044, + "fcm_dpo/margin": 120.28823852539062, + "fcm_dpo/q_t": 0.3560749888420105, + "grad_norm": 52.943965911865234, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.05986308678984642, + "logits/rejected": -0.04425333812832832, + "logps/chosen": -255.14764404296875, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -395.6673583984375, + "loss": 0.9605, + "margin_dpo/margin_mean": 120.28823852539062, + "margin_dpo/margin_std": 152.96563720703125, + "step": 303 + }, + { + "KL/chosen_KL_mean": -165.98196411132812, + "KL/mean": -233.64004516601562, + "KL/rejected_KL_mean": -301.2981262207031, + "KL/std": 135.4745635986328, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.005975336767733097, + "fcm_dpo/delta": -0.41170263290405273, + "fcm_dpo/margin": 135.31613159179688, + "fcm_dpo/q_t": 0.3234283924102783, + "grad_norm": 53.222232818603516, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.05158793553709984, + "logits/rejected": -0.03388179466128349, + "logps/chosen": -237.93267822265625, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -391.7701416015625, + "loss": 0.8287, + "margin_dpo/margin_mean": 135.31613159179688, + "margin_dpo/margin_std": 107.90045166015625, + "step": 304 + }, + { + "KL/chosen_KL_mean": -180.7779541015625, + "KL/mean": -243.37786865234375, + "KL/rejected_KL_mean": -305.9777526855469, + "KL/std": 146.08969116210938, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.00592338852584362, + "fcm_dpo/delta": -0.34457629919052124, + "fcm_dpo/margin": 125.19982147216797, + "fcm_dpo/q_t": 0.34956198930740356, + "grad_norm": 42.523590087890625, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.07274527102708817, + "logits/rejected": -0.05266021192073822, + "logps/chosen": -247.5731964111328, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -398.73236083984375, + "loss": 0.9331, + "margin_dpo/margin_mean": 125.19982147216797, + "margin_dpo/margin_std": 148.1181640625, + "step": 305 + }, + { + "KL/chosen_KL_mean": -166.89993286132812, + "KL/mean": -229.02658081054688, + "KL/rejected_KL_mean": -291.1532287597656, + "KL/std": 123.83683776855469, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.005888267885893583, + "fcm_dpo/delta": -0.33409184217453003, + "fcm_dpo/margin": 124.2532958984375, + "fcm_dpo/q_t": 0.3439217805862427, + "grad_norm": 33.7769889831543, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.051158562302589417, + "logits/rejected": -0.02742909826338291, + "logps/chosen": -236.58383178710938, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -376.31243896484375, + "loss": 0.8957, + "margin_dpo/margin_mean": 124.2532958984375, + "margin_dpo/margin_std": 126.36758422851562, + "step": 306 + }, + { + "KL/chosen_KL_mean": -148.6228790283203, + "KL/mean": -208.16824340820312, + "KL/rejected_KL_mean": -267.713623046875, + "KL/std": 111.05006408691406, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.005849805660545826, + "fcm_dpo/delta": -0.2989043593406677, + "fcm_dpo/margin": 119.09072875976562, + "fcm_dpo/q_t": 0.35078108310699463, + "grad_norm": 34.638851165771484, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.06943367421627045, + "logits/rejected": -0.05149272829294205, + "logps/chosen": -218.78829956054688, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -354.6859130859375, + "loss": 0.9325, + "margin_dpo/margin_mean": 119.09072875976562, + "margin_dpo/margin_std": 130.987548828125, + "step": 307 + }, + { + "KL/chosen_KL_mean": -148.8807830810547, + "KL/mean": -215.55865478515625, + "KL/rejected_KL_mean": -282.23651123046875, + "KL/std": 124.94607543945312, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.005810791626572609, + "fcm_dpo/delta": -0.37781667709350586, + "fcm_dpo/margin": 133.3557586669922, + "fcm_dpo/q_t": 0.3337476849555969, + "grad_norm": 49.99055480957031, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.09315188974142075, + "logits/rejected": -0.07270434498786926, + "logps/chosen": -204.1257781982422, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -361.6087951660156, + "loss": 0.8706, + "margin_dpo/margin_mean": 133.35574340820312, + "margin_dpo/margin_std": 125.37275695800781, + "step": 308 + }, + { + "KL/chosen_KL_mean": -157.6322479248047, + "KL/mean": -224.00604248046875, + "KL/rejected_KL_mean": -290.3798522949219, + "KL/std": 127.0302505493164, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0057712579146027565, + "fcm_dpo/delta": -0.3687945008277893, + "fcm_dpo/margin": 132.74758911132812, + "fcm_dpo/q_t": 0.3385184407234192, + "grad_norm": 59.66086959838867, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.023555653169751167, + "logits/rejected": -0.007646533194929361, + "logps/chosen": -206.59133911132812, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -372.7205810546875, + "loss": 0.8871, + "margin_dpo/margin_mean": 132.74758911132812, + "margin_dpo/margin_std": 133.69052124023438, + "step": 309 + }, + { + "KL/chosen_KL_mean": -188.2933349609375, + "KL/mean": -252.32266235351562, + "KL/rejected_KL_mean": -316.3519592285156, + "KL/std": 112.0794448852539, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.005731599871069193, + "fcm_dpo/delta": -0.3363185226917267, + "fcm_dpo/margin": 128.05862426757812, + "fcm_dpo/q_t": 0.3372734785079956, + "grad_norm": 72.6383056640625, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.004752453416585922, + "logits/rejected": 0.014258839190006256, + "logps/chosen": -251.03512573242188, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -396.28228759765625, + "loss": 0.8615, + "margin_dpo/margin_mean": 128.05862426757812, + "margin_dpo/margin_std": 103.54080200195312, + "step": 310 + }, + { + "KL/chosen_KL_mean": -207.9415283203125, + "KL/mean": -282.1041259765625, + "KL/rejected_KL_mean": -356.2667541503906, + "KL/std": 153.33126831054688, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.005678868852555752, + "fcm_dpo/delta": -0.44619858264923096, + "fcm_dpo/margin": 148.32522583007812, + "fcm_dpo/q_t": 0.3302444815635681, + "grad_norm": 41.24656295776367, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.026333574205636978, + "logits/rejected": -0.006447825580835342, + "logps/chosen": -260.9695129394531, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -433.7049560546875, + "loss": 0.8681, + "margin_dpo/margin_mean": 148.32521057128906, + "margin_dpo/margin_std": 156.83599853515625, + "step": 311 + }, + { + "KL/chosen_KL_mean": -199.4840850830078, + "KL/mean": -265.8037109375, + "KL/rejected_KL_mean": -332.12335205078125, + "KL/std": 129.70655822753906, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0056402478367090225, + "fcm_dpo/delta": -0.3506297469139099, + "fcm_dpo/margin": 132.63925170898438, + "fcm_dpo/q_t": 0.3427596688270569, + "grad_norm": 41.71390151977539, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": 0.07399940490722656, + "logits/rejected": 0.08698739111423492, + "logps/chosen": -248.87631225585938, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -407.9161376953125, + "loss": 0.9045, + "margin_dpo/margin_mean": 132.63925170898438, + "margin_dpo/margin_std": 142.93551635742188, + "step": 312 + }, + { + "KL/chosen_KL_mean": -188.4248046875, + "KL/mean": -250.8129119873047, + "KL/rejected_KL_mean": -313.2010192871094, + "KL/std": 134.4412841796875, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.005608034785836935, + "fcm_dpo/delta": -0.30175814032554626, + "fcm_dpo/margin": 124.77619171142578, + "fcm_dpo/q_t": 0.3542812466621399, + "grad_norm": 41.585113525390625, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": 0.05724754184484482, + "logits/rejected": 0.06064663082361221, + "logps/chosen": -238.57754516601562, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -399.60723876953125, + "loss": 0.9593, + "margin_dpo/margin_mean": 124.77619171142578, + "margin_dpo/margin_std": 149.64382934570312, + "step": 313 + }, + { + "KL/chosen_KL_mean": -205.89181518554688, + "KL/mean": -266.6488037109375, + "KL/rejected_KL_mean": -327.40576171875, + "KL/std": 134.92982482910156, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.005567425861954689, + "fcm_dpo/delta": -0.278642863035202, + "fcm_dpo/margin": 121.51395416259766, + "fcm_dpo/q_t": 0.36058565974235535, + "grad_norm": 32.8221435546875, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.02883746661245823, + "logits/rejected": -0.026101484894752502, + "logps/chosen": -263.12939453125, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -425.0023193359375, + "loss": 0.9767, + "margin_dpo/margin_mean": 121.51395416259766, + "margin_dpo/margin_std": 155.44680786132812, + "step": 314 + }, + { + "KL/chosen_KL_mean": -199.2986297607422, + "KL/mean": -259.0294189453125, + "KL/rejected_KL_mean": -318.76019287109375, + "KL/std": 114.86151885986328, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.005539732985198498, + "fcm_dpo/delta": -0.26360365748405457, + "fcm_dpo/margin": 119.46156311035156, + "fcm_dpo/q_t": 0.3567544221878052, + "grad_norm": 74.74089050292969, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": 0.07405808568000793, + "logits/rejected": 0.07599302381277084, + "logps/chosen": -248.36822509765625, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -404.4410705566406, + "loss": 0.9342, + "margin_dpo/margin_mean": 119.46156311035156, + "margin_dpo/margin_std": 126.41343688964844, + "step": 315 + }, + { + "KL/chosen_KL_mean": -188.63063049316406, + "KL/mean": -265.5641784667969, + "KL/rejected_KL_mean": -342.49774169921875, + "KL/std": 138.33914184570312, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.005493114702403545, + "fcm_dpo/delta": -0.44937270879745483, + "fcm_dpo/margin": 153.8671112060547, + "fcm_dpo/q_t": 0.3254196047782898, + "grad_norm": 40.97710418701172, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": 0.024657126516103745, + "logits/rejected": 0.02749829925596714, + "logps/chosen": -242.89138793945312, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -443.7792053222656, + "loss": 0.8376, + "margin_dpo/margin_mean": 153.8671112060547, + "margin_dpo/margin_std": 144.87255859375, + "step": 316 + }, + { + "KL/chosen_KL_mean": -188.8997344970703, + "KL/mean": -271.1233825683594, + "KL/rejected_KL_mean": -353.3470458984375, + "KL/std": 128.29502868652344, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.005445053800940514, + "fcm_dpo/delta": -0.499763548374176, + "fcm_dpo/margin": 164.44729614257812, + "fcm_dpo/q_t": 0.31016016006469727, + "grad_norm": 50.394901275634766, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.009579016827046871, + "logits/rejected": 0.005471055395901203, + "logps/chosen": -244.9939422607422, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -454.04608154296875, + "loss": 0.7925, + "margin_dpo/margin_mean": 164.44729614257812, + "margin_dpo/margin_std": 132.87127685546875, + "step": 317 + }, + { + "KL/chosen_KL_mean": -189.861572265625, + "KL/mean": -268.44500732421875, + "KL/rejected_KL_mean": -347.0284423828125, + "KL/std": 138.30384826660156, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.005394988227635622, + "fcm_dpo/delta": -0.4516098201274872, + "fcm_dpo/margin": 157.1668701171875, + "fcm_dpo/q_t": 0.32090169191360474, + "grad_norm": 45.80780029296875, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.07158035039901733, + "logits/rejected": -0.04287775605916977, + "logps/chosen": -254.50726318359375, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -429.7926940917969, + "loss": 0.824, + "margin_dpo/margin_mean": 157.1668701171875, + "margin_dpo/margin_std": 138.86419677734375, + "step": 318 + }, + { + "KL/chosen_KL_mean": -201.04327392578125, + "KL/mean": -275.26953125, + "KL/rejected_KL_mean": -349.4957275390625, + "KL/std": 147.4473876953125, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.005349372513592243, + "fcm_dpo/delta": -0.3972519040107727, + "fcm_dpo/margin": 148.45248413085938, + "fcm_dpo/q_t": 0.332307368516922, + "grad_norm": 34.4515266418457, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": 0.026266800239682198, + "logits/rejected": 0.012189566157758236, + "logps/chosen": -250.42703247070312, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -463.4022521972656, + "loss": 0.8673, + "margin_dpo/margin_mean": 148.45248413085938, + "margin_dpo/margin_std": 146.03314208984375, + "step": 319 + }, + { + "KL/chosen_KL_mean": -222.93658447265625, + "KL/mean": -306.93267822265625, + "KL/rejected_KL_mean": -390.9288330078125, + "KL/std": 164.26864624023438, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.005297025199979544, + "fcm_dpo/delta": -0.4944573640823364, + "fcm_dpo/margin": 167.99224853515625, + "fcm_dpo/q_t": 0.31936994194984436, + "grad_norm": 39.490623474121094, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": 0.03660593554377556, + "logits/rejected": 0.03375482186675072, + "logps/chosen": -282.44146728515625, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -488.5959777832031, + "loss": 0.8279, + "margin_dpo/margin_mean": 167.99224853515625, + "margin_dpo/margin_std": 165.0037841796875, + "step": 320 + }, + { + "KL/chosen_KL_mean": -280.66668701171875, + "KL/mean": -353.94921875, + "KL/rejected_KL_mean": -427.2317199707031, + "KL/std": 163.1666259765625, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.005250557325780392, + "fcm_dpo/delta": -0.37306004762649536, + "fcm_dpo/margin": 146.5650634765625, + "fcm_dpo/q_t": 0.3445137143135071, + "grad_norm": 56.03080749511719, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": 0.18314793705940247, + "logits/rejected": 0.18638217449188232, + "logps/chosen": -342.2153625488281, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -518.8727416992188, + "loss": 0.9378, + "margin_dpo/margin_mean": 146.5650634765625, + "margin_dpo/margin_std": 174.0413055419922, + "step": 321 + }, + { + "KL/chosen_KL_mean": -262.09796142578125, + "KL/mean": -339.8280334472656, + "KL/rejected_KL_mean": -417.55804443359375, + "KL/std": 156.5028839111328, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.005210362374782562, + "fcm_dpo/delta": -0.4137437343597412, + "fcm_dpo/margin": 155.4600830078125, + "fcm_dpo/q_t": 0.3320925831794739, + "grad_norm": 55.58685302734375, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": 0.1899016797542572, + "logits/rejected": 0.19282090663909912, + "logps/chosen": -319.38800048828125, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -513.3079833984375, + "loss": 0.8761, + "margin_dpo/margin_mean": 155.4600830078125, + "margin_dpo/margin_std": 157.8816375732422, + "step": 322 + }, + { + "KL/chosen_KL_mean": -267.2202453613281, + "KL/mean": -341.1400451660156, + "KL/rejected_KL_mean": -415.05987548828125, + "KL/std": 155.8050994873047, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.005172400735318661, + "fcm_dpo/delta": -0.36769282817840576, + "fcm_dpo/margin": 147.839599609375, + "fcm_dpo/q_t": 0.3415544033050537, + "grad_norm": 143.9473114013672, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": 0.11291222274303436, + "logits/rejected": 0.11311540007591248, + "logps/chosen": -318.4542236328125, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -490.1217956542969, + "loss": 0.9167, + "margin_dpo/margin_mean": 147.839599609375, + "margin_dpo/margin_std": 162.0154266357422, + "step": 323 + }, + { + "KL/chosen_KL_mean": -295.0044250488281, + "KL/mean": -371.0313720703125, + "KL/rejected_KL_mean": -447.05828857421875, + "KL/std": 164.284912109375, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.0051414198242127895, + "fcm_dpo/delta": -0.3846050798892975, + "fcm_dpo/margin": 152.05386352539062, + "fcm_dpo/q_t": 0.3421197533607483, + "grad_norm": 151.77749633789062, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": 0.07491143047809601, + "logits/rejected": 0.09102797508239746, + "logps/chosen": -360.13958740234375, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -533.5357666015625, + "loss": 0.9323, + "margin_dpo/margin_mean": 152.05386352539062, + "margin_dpo/margin_std": 174.98020935058594, + "step": 324 + }, + { + "KL/chosen_KL_mean": -228.12307739257812, + "KL/mean": -293.42010498046875, + "KL/rejected_KL_mean": -358.7171630859375, + "KL/std": 157.68438720703125, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.005104456562548876, + "fcm_dpo/delta": -0.2684288024902344, + "fcm_dpo/margin": 130.59408569335938, + "fcm_dpo/q_t": 0.3565933108329773, + "grad_norm": 39.878387451171875, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": 0.0017247714567929506, + "logits/rejected": 0.029396656900644302, + "logps/chosen": -284.33868408203125, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -428.8031005859375, + "loss": 0.9423, + "margin_dpo/margin_mean": 130.59408569335938, + "margin_dpo/margin_std": 139.77975463867188, + "step": 325 + }, + { + "KL/chosen_KL_mean": -236.8380126953125, + "KL/mean": -300.6353759765625, + "KL/rejected_KL_mean": -364.4327392578125, + "KL/std": 140.99322509765625, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.005078763701021671, + "fcm_dpo/delta": -0.24965548515319824, + "fcm_dpo/margin": 127.59471893310547, + "fcm_dpo/q_t": 0.35963454842567444, + "grad_norm": 53.66951370239258, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.02369173988699913, + "logits/rejected": 0.006208531558513641, + "logps/chosen": -309.56298828125, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -444.2795104980469, + "loss": 0.9444, + "margin_dpo/margin_mean": 127.59471893310547, + "margin_dpo/margin_std": 138.0997772216797, + "step": 326 + }, + { + "KL/chosen_KL_mean": -214.43966674804688, + "KL/mean": -307.7969665527344, + "KL/rejected_KL_mean": -401.154296875, + "KL/std": 162.09494018554688, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.005029140971601009, + "fcm_dpo/delta": -0.5444232821464539, + "fcm_dpo/margin": 186.71456909179688, + "fcm_dpo/q_t": 0.3074370324611664, + "grad_norm": 54.93585968017578, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.056819118559360504, + "logits/rejected": -0.03858664631843567, + "logps/chosen": -283.5740966796875, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -513.0880737304688, + "loss": 0.8057, + "margin_dpo/margin_mean": 186.71456909179688, + "margin_dpo/margin_std": 171.67691040039062, + "step": 327 + }, + { + "KL/chosen_KL_mean": -220.3560333251953, + "KL/mean": -298.1620178222656, + "KL/rejected_KL_mean": -375.968017578125, + "KL/std": 156.8230743408203, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.004990137182176113, + "fcm_dpo/delta": -0.37941503524780273, + "fcm_dpo/margin": 155.61196899414062, + "fcm_dpo/q_t": 0.3383353352546692, + "grad_norm": 50.86262893676758, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.006911966949701309, + "logits/rejected": -0.007989497855305672, + "logps/chosen": -280.043212890625, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -466.822998046875, + "loss": 0.882, + "margin_dpo/margin_mean": 155.61196899414062, + "margin_dpo/margin_std": 160.84292602539062, + "step": 328 + }, + { + "KL/chosen_KL_mean": -244.30880737304688, + "KL/mean": -326.5146484375, + "KL/rejected_KL_mean": -408.720458984375, + "KL/std": 164.2136688232422, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.004945470951497555, + "fcm_dpo/delta": -0.4168541431427002, + "fcm_dpo/margin": 164.41168212890625, + "fcm_dpo/q_t": 0.3338143825531006, + "grad_norm": 45.79575729370117, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": 0.0429445244371891, + "logits/rejected": 0.052189555019140244, + "logps/chosen": -309.55499267578125, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -509.4181823730469, + "loss": 0.8922, + "margin_dpo/margin_mean": 164.4116668701172, + "margin_dpo/margin_std": 175.97177124023438, + "step": 329 + }, + { + "KL/chosen_KL_mean": -212.8704376220703, + "KL/mean": -296.4083251953125, + "KL/rejected_KL_mean": -379.94622802734375, + "KL/std": 165.13101196289062, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.004906678572297096, + "fcm_dpo/delta": -0.42331045866012573, + "fcm_dpo/margin": 167.07577514648438, + "fcm_dpo/q_t": 0.3293718695640564, + "grad_norm": 37.50328063964844, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": 0.06113366037607193, + "logits/rejected": 0.05446569621562958, + "logps/chosen": -259.8687744140625, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -466.82305908203125, + "loss": 0.8674, + "margin_dpo/margin_mean": 167.07577514648438, + "margin_dpo/margin_std": 169.67691040039062, + "step": 330 + }, + { + "KL/chosen_KL_mean": -211.05227661132812, + "KL/mean": -298.08056640625, + "KL/rejected_KL_mean": -385.10882568359375, + "KL/std": 136.6324462890625, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.004862654022872448, + "fcm_dpo/delta": -0.4503188729286194, + "fcm_dpo/margin": 174.05654907226562, + "fcm_dpo/q_t": 0.3146325349807739, + "grad_norm": 35.09370422363281, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": 0.08672450482845306, + "logits/rejected": 0.0972529947757721, + "logps/chosen": -261.5765380859375, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -474.124267578125, + "loss": 0.787, + "margin_dpo/margin_mean": 174.05654907226562, + "margin_dpo/margin_std": 125.04080200195312, + "step": 331 + }, + { + "KL/chosen_KL_mean": -221.8607635498047, + "KL/mean": -282.05712890625, + "KL/rejected_KL_mean": -342.2535095214844, + "KL/std": 135.83778381347656, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.004840066656470299, + "fcm_dpo/delta": -0.18376365303993225, + "fcm_dpo/margin": 120.39274597167969, + "fcm_dpo/q_t": 0.3716254234313965, + "grad_norm": 37.009132385253906, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": 0.06577952951192856, + "logits/rejected": 0.07150351256132126, + "logps/chosen": -271.0410461425781, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -418.7386474609375, + "loss": 0.9793, + "margin_dpo/margin_mean": 120.39274597167969, + "margin_dpo/margin_std": 134.56973266601562, + "step": 332 + }, + { + "KL/chosen_KL_mean": -219.58956909179688, + "KL/mean": -301.25714111328125, + "KL/rejected_KL_mean": -382.92474365234375, + "KL/std": 151.05551147460938, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.00480304891243577, + "fcm_dpo/delta": -0.3880183696746826, + "fcm_dpo/margin": 163.33514404296875, + "fcm_dpo/q_t": 0.3381913900375366, + "grad_norm": 59.359169006347656, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.005021329037845135, + "logits/rejected": 0.0012002307921648026, + "logps/chosen": -283.3453063964844, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -477.9688415527344, + "loss": 0.8956, + "margin_dpo/margin_mean": 163.3351593017578, + "margin_dpo/margin_std": 170.32351684570312, + "step": 333 + }, + { + "KL/chosen_KL_mean": -213.5518798828125, + "KL/mean": -287.912109375, + "KL/rejected_KL_mean": -362.2723693847656, + "KL/std": 166.07781982421875, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.004774728324264288, + "fcm_dpo/delta": -0.3123623728752136, + "fcm_dpo/margin": 148.72048950195312, + "fcm_dpo/q_t": 0.3478446304798126, + "grad_norm": 47.94345474243164, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.033267103135585785, + "logits/rejected": -0.024581972509622574, + "logps/chosen": -280.5316162109375, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -457.58929443359375, + "loss": 0.9224, + "margin_dpo/margin_mean": 148.72048950195312, + "margin_dpo/margin_std": 158.16165161132812, + "step": 334 + }, + { + "KL/chosen_KL_mean": -234.0174560546875, + "KL/mean": -293.8703308105469, + "KL/rejected_KL_mean": -353.72320556640625, + "KL/std": 146.930908203125, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.004757601767778397, + "fcm_dpo/delta": -0.17044730484485626, + "fcm_dpo/margin": 119.70578002929688, + "fcm_dpo/q_t": 0.3778232932090759, + "grad_norm": 61.829471588134766, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": 0.06888309121131897, + "logits/rejected": 0.08831358700990677, + "logps/chosen": -296.5599365234375, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -441.3409423828125, + "loss": 1.0199, + "margin_dpo/margin_mean": 119.70578002929688, + "margin_dpo/margin_std": 158.07931518554688, + "step": 335 + }, + { + "KL/chosen_KL_mean": -227.72152709960938, + "KL/mean": -309.95440673828125, + "KL/rejected_KL_mean": -392.187255859375, + "KL/std": 163.75076293945312, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.004724455066025257, + "fcm_dpo/delta": -0.3799822926521301, + "fcm_dpo/margin": 164.46572875976562, + "fcm_dpo/q_t": 0.3370096683502197, + "grad_norm": 45.03193283081055, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": 0.044198669493198395, + "logits/rejected": 0.048565879464149475, + "logps/chosen": -282.252685546875, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -496.59149169921875, + "loss": 0.9, + "margin_dpo/margin_mean": 164.46572875976562, + "margin_dpo/margin_std": 176.58241271972656, + "step": 336 + }, + { + "KL/chosen_KL_mean": -188.8308868408203, + "KL/mean": -276.39520263671875, + "KL/rejected_KL_mean": -363.9595642089844, + "KL/std": 147.94992065429688, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.004683498293161392, + "fcm_dpo/delta": -0.4239504933357239, + "fcm_dpo/margin": 175.12867736816406, + "fcm_dpo/q_t": 0.32411181926727295, + "grad_norm": 44.84009552001953, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": 0.023412253707647324, + "logits/rejected": 0.031911641359329224, + "logps/chosen": -253.95957946777344, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -465.68658447265625, + "loss": 0.835, + "margin_dpo/margin_mean": 175.12869262695312, + "margin_dpo/margin_std": 152.6188201904297, + "step": 337 + }, + { + "KL/chosen_KL_mean": -193.21914672851562, + "KL/mean": -276.388916015625, + "KL/rejected_KL_mean": -359.5587158203125, + "KL/std": 152.16172790527344, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.004649758338928223, + "fcm_dpo/delta": -0.3763352632522583, + "fcm_dpo/margin": 166.3395233154297, + "fcm_dpo/q_t": 0.33298492431640625, + "grad_norm": 31.02379608154297, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": 0.01635255478322506, + "logits/rejected": 0.037509627640247345, + "logps/chosen": -251.64186096191406, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -448.6272277832031, + "loss": 0.8561, + "margin_dpo/margin_mean": 166.33953857421875, + "margin_dpo/margin_std": 148.49234008789062, + "step": 338 + }, + { + "KL/chosen_KL_mean": -203.98532104492188, + "KL/mean": -286.79119873046875, + "KL/rejected_KL_mean": -369.59710693359375, + "KL/std": 155.88446044921875, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.00461919791996479, + "fcm_dpo/delta": -0.3676402270793915, + "fcm_dpo/margin": 165.61180114746094, + "fcm_dpo/q_t": 0.3345920443534851, + "grad_norm": 38.68682861328125, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": 0.03661680966615677, + "logits/rejected": 0.04869178682565689, + "logps/chosen": -263.9806213378906, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -473.508056640625, + "loss": 0.8596, + "margin_dpo/margin_mean": 165.61181640625, + "margin_dpo/margin_std": 143.4925079345703, + "step": 339 + }, + { + "KL/chosen_KL_mean": -226.04678344726562, + "KL/mean": -302.1143798828125, + "KL/rejected_KL_mean": -378.1820068359375, + "KL/std": 147.6373291015625, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.004583721049129963, + "fcm_dpo/delta": -0.2996503710746765, + "fcm_dpo/margin": 152.13523864746094, + "fcm_dpo/q_t": 0.35091155767440796, + "grad_norm": 48.29704284667969, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": 0.05171532928943634, + "logits/rejected": 0.07620946317911148, + "logps/chosen": -278.87701416015625, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -451.28924560546875, + "loss": 0.9251, + "margin_dpo/margin_mean": 152.13523864746094, + "margin_dpo/margin_std": 163.4954833984375, + "step": 340 + }, + { + "KL/chosen_KL_mean": -225.3345947265625, + "KL/mean": -312.0306396484375, + "KL/rejected_KL_mean": -398.72674560546875, + "KL/std": 159.55081176757812, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.004550696816295385, + "fcm_dpo/delta": -0.3923296332359314, + "fcm_dpo/margin": 173.39215087890625, + "fcm_dpo/q_t": 0.3318275809288025, + "grad_norm": 40.54072570800781, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": 0.03106379508972168, + "logits/rejected": 0.02473168633878231, + "logps/chosen": -273.234375, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -500.53662109375, + "loss": 0.8659, + "margin_dpo/margin_mean": 173.39215087890625, + "margin_dpo/margin_std": 164.19363403320312, + "step": 341 + }, + { + "KL/chosen_KL_mean": -217.1148681640625, + "KL/mean": -297.3704833984375, + "KL/rejected_KL_mean": -377.6260986328125, + "KL/std": 156.92709350585938, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.004523161798715591, + "fcm_dpo/delta": -0.3282979726791382, + "fcm_dpo/margin": 160.51124572753906, + "fcm_dpo/q_t": 0.3459194302558899, + "grad_norm": 38.68334197998047, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.016471415758132935, + "logits/rejected": 0.007698075845837593, + "logps/chosen": -289.11151123046875, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -470.2156982421875, + "loss": 0.907, + "margin_dpo/margin_mean": 160.51123046875, + "margin_dpo/margin_std": 168.6878662109375, + "step": 342 + }, + { + "KL/chosen_KL_mean": -210.16558837890625, + "KL/mean": -303.23236083984375, + "KL/rejected_KL_mean": -396.29913330078125, + "KL/std": 167.3316650390625, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.004483702592551708, + "fcm_dpo/delta": -0.4382782280445099, + "fcm_dpo/margin": 186.133544921875, + "fcm_dpo/q_t": 0.31999701261520386, + "grad_norm": 34.39127731323242, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": 0.0795452892780304, + "logits/rejected": 0.08556081354618073, + "logps/chosen": -264.57122802734375, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -507.340576171875, + "loss": 0.8144, + "margin_dpo/margin_mean": 186.13356018066406, + "margin_dpo/margin_std": 149.89895629882812, + "step": 343 + }, + { + "KL/chosen_KL_mean": -224.2109375, + "KL/mean": -304.8324890136719, + "KL/rejected_KL_mean": -385.45404052734375, + "KL/std": 175.81173706054688, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.004457741975784302, + "fcm_dpo/delta": -0.32097604870796204, + "fcm_dpo/margin": 161.2430877685547, + "fcm_dpo/q_t": 0.34674492478370667, + "grad_norm": 35.81315994262695, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": 0.07259823381900787, + "logits/rejected": 0.07393225282430649, + "logps/chosen": -278.17559814453125, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -476.077392578125, + "loss": 0.9048, + "margin_dpo/margin_mean": 161.2430877685547, + "margin_dpo/margin_std": 159.20733642578125, + "step": 344 + }, + { + "KL/chosen_KL_mean": -265.6514587402344, + "KL/mean": -350.68133544921875, + "KL/rejected_KL_mean": -435.711181640625, + "KL/std": 172.37249755859375, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.004424700513482094, + "fcm_dpo/delta": -0.3549978733062744, + "fcm_dpo/margin": 170.05975341796875, + "fcm_dpo/q_t": 0.34145864844322205, + "grad_norm": 38.0777473449707, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": 0.026162656024098396, + "logits/rejected": 0.039617687463760376, + "logps/chosen": -327.337158203125, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -535.2015991210938, + "loss": 0.8995, + "margin_dpo/margin_mean": 170.05975341796875, + "margin_dpo/margin_std": 177.85409545898438, + "step": 345 + }, + { + "KL/chosen_KL_mean": -273.33416748046875, + "KL/mean": -353.783203125, + "KL/rejected_KL_mean": -434.23223876953125, + "KL/std": 161.6199951171875, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.004392815753817558, + "fcm_dpo/delta": -0.30906641483306885, + "fcm_dpo/margin": 160.89810180664062, + "fcm_dpo/q_t": 0.34564802050590515, + "grad_norm": 52.034271240234375, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": 0.1980515569448471, + "logits/rejected": 0.2146371304988861, + "logps/chosen": -328.5904235839844, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -511.6475524902344, + "loss": 0.9115, + "margin_dpo/margin_mean": 160.89810180664062, + "margin_dpo/margin_std": 162.71728515625, + "step": 346 + }, + { + "KL/chosen_KL_mean": -277.8858642578125, + "KL/mean": -360.41400146484375, + "KL/rejected_KL_mean": -442.942138671875, + "KL/std": 175.3711395263672, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.004364816937595606, + "fcm_dpo/delta": -0.32285863161087036, + "fcm_dpo/margin": 165.0562744140625, + "fcm_dpo/q_t": 0.3494032621383667, + "grad_norm": 56.28925323486328, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": 0.15960478782653809, + "logits/rejected": 0.16570210456848145, + "logps/chosen": -335.4521179199219, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -535.2972412109375, + "loss": 0.9311, + "margin_dpo/margin_mean": 165.0562744140625, + "margin_dpo/margin_std": 186.4940643310547, + "step": 347 + }, + { + "KL/chosen_KL_mean": -251.89862060546875, + "KL/mean": -325.88287353515625, + "KL/rejected_KL_mean": -399.8670959472656, + "KL/std": 164.47723388671875, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.0043387385085225105, + "fcm_dpo/delta": -0.2439822554588318, + "fcm_dpo/margin": 147.9684600830078, + "fcm_dpo/q_t": 0.3626495897769928, + "grad_norm": 35.06428146362305, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": 0.08004555106163025, + "logits/rejected": 0.08033594489097595, + "logps/chosen": -308.21630859375, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -489.0054626464844, + "loss": 0.9631, + "margin_dpo/margin_mean": 147.9684600830078, + "margin_dpo/margin_std": 167.98294067382812, + "step": 348 + }, + { + "KL/chosen_KL_mean": -244.73312377929688, + "KL/mean": -323.46240234375, + "KL/rejected_KL_mean": -402.1917419433594, + "KL/std": 166.6591033935547, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.004322429653257132, + "fcm_dpo/delta": -0.2824273109436035, + "fcm_dpo/margin": 157.45860290527344, + "fcm_dpo/q_t": 0.35909608006477356, + "grad_norm": 33.48833465576172, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": 0.15586219727993011, + "logits/rejected": 0.1613733470439911, + "logps/chosen": -302.7586364746094, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -499.6968994140625, + "loss": 0.969, + "margin_dpo/margin_mean": 157.45860290527344, + "margin_dpo/margin_std": 195.89410400390625, + "step": 349 + }, + { + "KL/chosen_KL_mean": -244.75975036621094, + "KL/mean": -328.8629150390625, + "KL/rejected_KL_mean": -412.966064453125, + "KL/std": 161.11859130859375, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.004289613105356693, + "fcm_dpo/delta": -0.324157178401947, + "fcm_dpo/margin": 168.2063446044922, + "fcm_dpo/q_t": 0.3439641296863556, + "grad_norm": 51.959922790527344, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": 0.06945175677537918, + "logits/rejected": 0.07429882138967514, + "logps/chosen": -309.0902404785156, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -502.83770751953125, + "loss": 0.9093, + "margin_dpo/margin_mean": 168.2063446044922, + "margin_dpo/margin_std": 170.51910400390625, + "step": 350 + }, + { + "KL/chosen_KL_mean": -240.32424926757812, + "KL/mean": -336.2489013671875, + "KL/rejected_KL_mean": -432.17352294921875, + "KL/std": 187.81045532226562, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0042584724724292755, + "fcm_dpo/delta": -0.4204143285751343, + "fcm_dpo/margin": 191.84930419921875, + "fcm_dpo/q_t": 0.33178192377090454, + "grad_norm": 38.992149353027344, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": 0.03077036328613758, + "logits/rejected": 0.039955027401447296, + "logps/chosen": -300.9963684082031, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -533.7389526367188, + "loss": 0.8795, + "margin_dpo/margin_mean": 191.84930419921875, + "margin_dpo/margin_std": 203.04605102539062, + "step": 351 + }, + { + "KL/chosen_KL_mean": -280.1048889160156, + "KL/mean": -341.05145263671875, + "KL/rejected_KL_mean": -401.998046875, + "KL/std": 172.1851043701172, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.004238965921103954, + "fcm_dpo/delta": -0.11748933792114258, + "fcm_dpo/margin": 121.89315795898438, + "fcm_dpo/q_t": 0.39063560962677, + "grad_norm": 65.35124969482422, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": 0.10109634697437286, + "logits/rejected": 0.13250567018985748, + "logps/chosen": -351.04833984375, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -478.6400146484375, + "loss": 1.075, + "margin_dpo/margin_mean": 121.89315795898438, + "margin_dpo/margin_std": 185.5072479248047, + "step": 352 + }, + { + "KL/chosen_KL_mean": -263.424072265625, + "KL/mean": -338.3148193359375, + "KL/rejected_KL_mean": -413.2055358886719, + "KL/std": 168.17733764648438, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.004221927374601364, + "fcm_dpo/delta": -0.23407524824142456, + "fcm_dpo/margin": 149.781494140625, + "fcm_dpo/q_t": 0.36673933267593384, + "grad_norm": 39.59394454956055, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": 0.14468395709991455, + "logits/rejected": 0.15805163979530334, + "logps/chosen": -321.81939697265625, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -493.54107666015625, + "loss": 0.9878, + "margin_dpo/margin_mean": 149.781494140625, + "margin_dpo/margin_std": 186.81732177734375, + "step": 353 + }, + { + "KL/chosen_KL_mean": -220.44287109375, + "KL/mean": -310.34423828125, + "KL/rejected_KL_mean": -400.24560546875, + "KL/std": 157.10821533203125, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.0041963448747992516, + "fcm_dpo/delta": -0.35728389024734497, + "fcm_dpo/margin": 179.80274963378906, + "fcm_dpo/q_t": 0.33374539017677307, + "grad_norm": 46.97367858886719, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": 0.1409774124622345, + "logits/rejected": 0.15768709778785706, + "logps/chosen": -280.245849609375, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -489.00311279296875, + "loss": 0.8526, + "margin_dpo/margin_mean": 179.80274963378906, + "margin_dpo/margin_std": 148.87442016601562, + "step": 354 + }, + { + "KL/chosen_KL_mean": -224.162353515625, + "KL/mean": -311.8890380859375, + "KL/rejected_KL_mean": -399.61572265625, + "KL/std": 172.086669921875, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.004167201928794384, + "fcm_dpo/delta": -0.3337364196777344, + "fcm_dpo/margin": 175.45333862304688, + "fcm_dpo/q_t": 0.3396884500980377, + "grad_norm": 56.26557159423828, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": 0.09514348208904266, + "logits/rejected": 0.11814308166503906, + "logps/chosen": -278.2908630371094, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -482.0217590332031, + "loss": 0.8716, + "margin_dpo/margin_mean": 175.45333862304688, + "margin_dpo/margin_std": 153.22659301757812, + "step": 355 + }, + { + "KL/chosen_KL_mean": -271.3259582519531, + "KL/mean": -326.330322265625, + "KL/rejected_KL_mean": -381.33465576171875, + "KL/std": 167.2308807373047, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.004161309450864792, + "fcm_dpo/delta": -0.058119527995586395, + "fcm_dpo/margin": 110.00871276855469, + "fcm_dpo/q_t": 0.4035111665725708, + "grad_norm": 47.588157653808594, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": 0.09138090908527374, + "logits/rejected": 0.1042904481291771, + "logps/chosen": -335.999755859375, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -457.23394775390625, + "loss": 1.1286, + "margin_dpo/margin_mean": 110.00870513916016, + "margin_dpo/margin_std": 191.81954956054688, + "step": 356 + }, + { + "KL/chosen_KL_mean": -258.8679504394531, + "KL/mean": -343.3545227050781, + "KL/rejected_KL_mean": -427.84112548828125, + "KL/std": 177.21685791015625, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.004135658964514732, + "fcm_dpo/delta": -0.30117738246917725, + "fcm_dpo/margin": 168.97312927246094, + "fcm_dpo/q_t": 0.3473314940929413, + "grad_norm": 43.65181350708008, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": 0.13779321312904358, + "logits/rejected": 0.15212051570415497, + "logps/chosen": -311.59375, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -514.6822509765625, + "loss": 0.9065, + "margin_dpo/margin_mean": 168.97314453125, + "margin_dpo/margin_std": 164.52825927734375, + "step": 357 + }, + { + "KL/chosen_KL_mean": -251.47955322265625, + "KL/mean": -322.9197998046875, + "KL/rejected_KL_mean": -394.3599853515625, + "KL/std": 168.0701904296875, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0041207182221114635, + "fcm_dpo/delta": -0.18986734747886658, + "fcm_dpo/margin": 142.88043212890625, + "fcm_dpo/q_t": 0.3721756041049957, + "grad_norm": 33.72293472290039, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": 0.0762886106967926, + "logits/rejected": 0.09385469555854797, + "logps/chosen": -314.68499755859375, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -482.7332763671875, + "loss": 0.9815, + "margin_dpo/margin_mean": 142.88043212890625, + "margin_dpo/margin_std": 166.3026123046875, + "step": 358 + }, + { + "KL/chosen_KL_mean": -285.80853271484375, + "KL/mean": -369.73028564453125, + "KL/rejected_KL_mean": -453.65203857421875, + "KL/std": 160.6344451904297, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.004096765071153641, + "fcm_dpo/delta": -0.2897596061229706, + "fcm_dpo/margin": 167.84347534179688, + "fcm_dpo/q_t": 0.355857789516449, + "grad_norm": 46.9790153503418, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": 0.2033603936433792, + "logits/rejected": 0.22514639794826508, + "logps/chosen": -342.1787414550781, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -535.8258056640625, + "loss": 0.9391, + "margin_dpo/margin_mean": 167.84347534179688, + "margin_dpo/margin_std": 188.49349975585938, + "step": 359 + }, + { + "KL/chosen_KL_mean": -284.4974670410156, + "KL/mean": -354.50091552734375, + "KL/rejected_KL_mean": -424.50439453125, + "KL/std": 144.01150512695312, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.004082887899130583, + "fcm_dpo/delta": -0.17258970439434052, + "fcm_dpo/margin": 140.0069122314453, + "fcm_dpo/q_t": 0.3715874254703522, + "grad_norm": 88.23158264160156, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": 0.22746475040912628, + "logits/rejected": 0.23542645573616028, + "logps/chosen": -335.9578552246094, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -494.34332275390625, + "loss": 0.9713, + "margin_dpo/margin_mean": 140.0069122314453, + "margin_dpo/margin_std": 142.29086303710938, + "step": 360 + }, + { + "KL/chosen_KL_mean": -290.6372985839844, + "KL/mean": -356.11724853515625, + "KL/rejected_KL_mean": -421.59716796875, + "KL/std": 156.99920654296875, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.004071630071848631, + "fcm_dpo/delta": -0.13391195237636566, + "fcm_dpo/margin": 130.95986938476562, + "fcm_dpo/q_t": 0.3829229176044464, + "grad_norm": 42.70572280883789, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": 0.18119779229164124, + "logits/rejected": 0.17793747782707214, + "logps/chosen": -344.5068054199219, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -512.3663940429688, + "loss": 1.026, + "margin_dpo/margin_mean": 130.95985412597656, + "margin_dpo/margin_std": 167.0465545654297, + "step": 361 + }, + { + "KL/chosen_KL_mean": -256.27581787109375, + "KL/mean": -353.5871887207031, + "KL/rejected_KL_mean": -450.8985290527344, + "KL/std": 164.17794799804688, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.004043214954435825, + "fcm_dpo/delta": -0.3900299668312073, + "fcm_dpo/margin": 194.62269592285156, + "fcm_dpo/q_t": 0.33122718334198, + "grad_norm": 36.74576950073242, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": 0.20381364226341248, + "logits/rejected": 0.19734057784080505, + "logps/chosen": -314.9148864746094, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -556.48046875, + "loss": 0.8476, + "margin_dpo/margin_mean": 194.62269592285156, + "margin_dpo/margin_std": 174.51174926757812, + "step": 362 + }, + { + "KL/chosen_KL_mean": -231.74911499023438, + "KL/mean": -337.29144287109375, + "KL/rejected_KL_mean": -442.833740234375, + "KL/std": 170.2020263671875, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.004010315518826246, + "fcm_dpo/delta": -0.4501842260360718, + "fcm_dpo/margin": 211.0846405029297, + "fcm_dpo/q_t": 0.31912097334861755, + "grad_norm": 42.53211975097656, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": 0.23380029201507568, + "logits/rejected": 0.24540770053863525, + "logps/chosen": -276.3074951171875, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -517.5286865234375, + "loss": 0.8198, + "margin_dpo/margin_mean": 211.08465576171875, + "margin_dpo/margin_std": 182.07489013671875, + "step": 363 + }, + { + "KL/chosen_KL_mean": -234.80836486816406, + "KL/mean": -312.7071533203125, + "KL/rejected_KL_mean": -390.60595703125, + "KL/std": 162.87295532226562, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.003988500684499741, + "fcm_dpo/delta": -0.22272124886512756, + "fcm_dpo/margin": 155.79754638671875, + "fcm_dpo/q_t": 0.3644035756587982, + "grad_norm": 41.610504150390625, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": 0.13949087262153625, + "logits/rejected": 0.13440436124801636, + "logps/chosen": -283.7030029296875, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -482.001708984375, + "loss": 0.9668, + "margin_dpo/margin_mean": 155.79754638671875, + "margin_dpo/margin_std": 175.89276123046875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -223.17138671875, + "KL/mean": -301.1219177246094, + "KL/rejected_KL_mean": -379.07244873046875, + "KL/std": 172.93638610839844, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.003969316370785236, + "fcm_dpo/delta": -0.22015851736068726, + "fcm_dpo/margin": 155.9010467529297, + "fcm_dpo/q_t": 0.36668330430984497, + "grad_norm": 34.92851638793945, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": 0.10566478222608566, + "logits/rejected": 0.09369112551212311, + "logps/chosen": -274.66412353515625, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -471.77410888671875, + "loss": 0.9583, + "margin_dpo/margin_mean": 155.9010467529297, + "margin_dpo/margin_std": 172.931396484375, + "step": 365 + }, + { + "KL/chosen_KL_mean": -193.80604553222656, + "KL/mean": -276.9259033203125, + "KL/rejected_KL_mean": -360.0457763671875, + "KL/std": 162.60922241210938, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0039481837302446365, + "fcm_dpo/delta": -0.25811201333999634, + "fcm_dpo/margin": 166.23971557617188, + "fcm_dpo/q_t": 0.3569996953010559, + "grad_norm": 33.496456146240234, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": 0.10501408576965332, + "logits/rejected": 0.10146645456552505, + "logps/chosen": -238.526611328125, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -443.3561706542969, + "loss": 0.9347, + "margin_dpo/margin_mean": 166.23971557617188, + "margin_dpo/margin_std": 172.90614318847656, + "step": 366 + }, + { + "KL/chosen_KL_mean": -196.26634216308594, + "KL/mean": -262.79730224609375, + "KL/rejected_KL_mean": -329.3282775878906, + "KL/std": 146.38552856445312, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.003936568275094032, + "fcm_dpo/delta": -0.12446263432502747, + "fcm_dpo/margin": 133.06192016601562, + "fcm_dpo/q_t": 0.38147711753845215, + "grad_norm": 33.32904052734375, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": 0.013883661478757858, + "logits/rejected": 0.0378500334918499, + "logps/chosen": -254.6717529296875, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -406.07958984375, + "loss": 1.005, + "margin_dpo/margin_mean": 133.06192016601562, + "margin_dpo/margin_std": 146.06251525878906, + "step": 367 + }, + { + "KL/chosen_KL_mean": -180.50881958007812, + "KL/mean": -272.2597961425781, + "KL/rejected_KL_mean": -364.0107116699219, + "KL/std": 155.9082794189453, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0039164479821920395, + "fcm_dpo/delta": -0.3209083676338196, + "fcm_dpo/margin": 183.5019073486328, + "fcm_dpo/q_t": 0.3380677103996277, + "grad_norm": 64.91632843017578, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": 0.005872292444109917, + "logits/rejected": 0.01200754102319479, + "logps/chosen": -224.96133422851562, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -462.56597900390625, + "loss": 0.8547, + "margin_dpo/margin_mean": 183.50192260742188, + "margin_dpo/margin_std": 136.1890869140625, + "step": 368 + }, + { + "KL/chosen_KL_mean": -213.70208740234375, + "KL/mean": -277.95220947265625, + "KL/rejected_KL_mean": -342.20233154296875, + "KL/std": 156.30667114257812, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0039048593025654554, + "fcm_dpo/delta": -0.10228417068719864, + "fcm_dpo/margin": 128.500244140625, + "fcm_dpo/q_t": 0.3887633681297302, + "grad_norm": 41.09241485595703, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": 0.05891437828540802, + "logits/rejected": 0.07951710373163223, + "logps/chosen": -285.0835876464844, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -433.4981689453125, + "loss": 1.0441, + "margin_dpo/margin_mean": 128.500244140625, + "margin_dpo/margin_std": 166.826416015625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -233.6761016845703, + "KL/mean": -288.24151611328125, + "KL/rejected_KL_mean": -342.8069152832031, + "KL/std": 150.2886962890625, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0039000860415399075, + "fcm_dpo/delta": -0.025729427114129066, + "fcm_dpo/margin": 109.13084411621094, + "fcm_dpo/q_t": 0.4031984806060791, + "grad_norm": 38.69504165649414, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": 0.10062532871961594, + "logits/rejected": 0.10277409851551056, + "logps/chosen": -305.2835998535156, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -440.06671142578125, + "loss": 1.0842, + "margin_dpo/margin_mean": 109.1308364868164, + "margin_dpo/margin_std": 149.1434326171875, + "step": 370 + }, + { + "KL/chosen_KL_mean": -227.7499542236328, + "KL/mean": -300.5270080566406, + "KL/rejected_KL_mean": -373.3040771484375, + "KL/std": 153.07830810546875, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.003890453139320016, + "fcm_dpo/delta": -0.1671864092350006, + "fcm_dpo/margin": 145.55413818359375, + "fcm_dpo/q_t": 0.3736514449119568, + "grad_norm": 35.500877380371094, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": 0.03682202100753784, + "logits/rejected": 0.0457378514111042, + "logps/chosen": -297.1644287109375, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -472.47625732421875, + "loss": 0.9946, + "margin_dpo/margin_mean": 145.55413818359375, + "margin_dpo/margin_std": 167.62692260742188, + "step": 371 + }, + { + "KL/chosen_KL_mean": -222.77537536621094, + "KL/mean": -321.4415283203125, + "KL/rejected_KL_mean": -420.107666015625, + "KL/std": 190.83251953125, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.0038639383856207132, + "fcm_dpo/delta": -0.36535942554473877, + "fcm_dpo/margin": 197.33230590820312, + "fcm_dpo/q_t": 0.3393678665161133, + "grad_norm": 46.464866638183594, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": 0.0988445058465004, + "logits/rejected": 0.1318180114030838, + "logps/chosen": -284.5933837890625, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -498.64715576171875, + "loss": 0.8871, + "margin_dpo/margin_mean": 197.33230590820312, + "margin_dpo/margin_std": 199.5469970703125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -251.97439575195312, + "KL/mean": -344.2081298828125, + "KL/rejected_KL_mean": -436.44189453125, + "KL/std": 179.76153564453125, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.003842129372060299, + "fcm_dpo/delta": -0.3108375668525696, + "fcm_dpo/margin": 184.46749877929688, + "fcm_dpo/q_t": 0.3436407446861267, + "grad_norm": 47.52699279785156, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": 0.12836693227291107, + "logits/rejected": 0.16331298649311066, + "logps/chosen": -316.1915283203125, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -522.4014892578125, + "loss": 0.8944, + "margin_dpo/margin_mean": 184.46749877929688, + "margin_dpo/margin_std": 168.44686889648438, + "step": 373 + }, + { + "KL/chosen_KL_mean": -236.91696166992188, + "KL/mean": -322.01025390625, + "KL/rejected_KL_mean": -407.1035461425781, + "KL/std": 194.6341552734375, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.003817587625235319, + "fcm_dpo/delta": -0.2515011727809906, + "fcm_dpo/margin": 170.18658447265625, + "fcm_dpo/q_t": 0.364173948764801, + "grad_norm": 34.61599349975586, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": 0.020081838592886925, + "logits/rejected": 0.05203642696142197, + "logps/chosen": -307.567138671875, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -500.74371337890625, + "loss": 0.9612, + "margin_dpo/margin_mean": 170.18658447265625, + "margin_dpo/margin_std": 200.57363891601562, + "step": 374 + }, + { + "KL/chosen_KL_mean": -278.1369323730469, + "KL/mean": -356.6187744140625, + "KL/rejected_KL_mean": -435.1006164550781, + "KL/std": 174.30870056152344, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0037989693228155375, + "fcm_dpo/delta": -0.1979631930589676, + "fcm_dpo/margin": 156.96368408203125, + "fcm_dpo/q_t": 0.37072139978408813, + "grad_norm": 33.315032958984375, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": 0.07743004709482193, + "logits/rejected": 0.08232336491346359, + "logps/chosen": -338.2171630859375, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -524.0389404296875, + "loss": 1.0017, + "margin_dpo/margin_mean": 156.96368408203125, + "margin_dpo/margin_std": 189.33087158203125, + "step": 375 + }, + { + "KL/chosen_KL_mean": -265.7765197753906, + "KL/mean": -362.2598876953125, + "KL/rejected_KL_mean": -458.7432861328125, + "KL/std": 179.02391052246094, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0037802434526383877, + "fcm_dpo/delta": -0.3319232761859894, + "fcm_dpo/margin": 192.9667510986328, + "fcm_dpo/q_t": 0.3432157039642334, + "grad_norm": 48.58695602416992, + "learning_rate": 2.5e-07, + "logits/chosen": 0.04117865860462189, + "logits/rejected": 0.06004368141293526, + "logps/chosen": -328.43682861328125, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -564.2698974609375, + "loss": 0.9002, + "margin_dpo/margin_mean": 192.9667510986328, + "margin_dpo/margin_std": 194.88125610351562, + "step": 376 + }, + { + "KL/chosen_KL_mean": -298.1991882324219, + "KL/mean": -388.93621826171875, + "KL/rejected_KL_mean": -479.67327880859375, + "KL/std": 186.58270263671875, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0037589063867926598, + "fcm_dpo/delta": -0.28404495120048523, + "fcm_dpo/margin": 181.47412109375, + "fcm_dpo/q_t": 0.3523944318294525, + "grad_norm": 38.296875, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": 0.1594271957874298, + "logits/rejected": 0.16929763555526733, + "logps/chosen": -352.67791748046875, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -578.3766479492188, + "loss": 0.929, + "margin_dpo/margin_mean": 181.47412109375, + "margin_dpo/margin_std": 191.09255981445312, + "step": 377 + }, + { + "KL/chosen_KL_mean": -306.78814697265625, + "KL/mean": -408.07318115234375, + "KL/rejected_KL_mean": -509.3581848144531, + "KL/std": 175.46084594726562, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.003731479635462165, + "fcm_dpo/delta": -0.3587607443332672, + "fcm_dpo/margin": 202.57005310058594, + "fcm_dpo/q_t": 0.333503395318985, + "grad_norm": 42.61096954345703, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": 0.27134329080581665, + "logits/rejected": 0.2833588719367981, + "logps/chosen": -351.80865478515625, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -597.4051513671875, + "loss": 0.8624, + "margin_dpo/margin_mean": 202.57003784179688, + "margin_dpo/margin_std": 175.72781372070312, + "step": 378 + }, + { + "KL/chosen_KL_mean": -331.314697265625, + "KL/mean": -430.47802734375, + "KL/rejected_KL_mean": -529.6412963867188, + "KL/std": 185.25119018554688, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.003705243580043316, + "fcm_dpo/delta": -0.3375876843929291, + "fcm_dpo/margin": 198.3266143798828, + "fcm_dpo/q_t": 0.344668984413147, + "grad_norm": 48.29153060913086, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": 0.18467631936073303, + "logits/rejected": 0.1723889261484146, + "logps/chosen": -384.4967956542969, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -643.9414672851562, + "loss": 0.909, + "margin_dpo/margin_mean": 198.32662963867188, + "margin_dpo/margin_std": 209.97447204589844, + "step": 379 + }, + { + "KL/chosen_KL_mean": -330.01873779296875, + "KL/mean": -440.7161560058594, + "KL/rejected_KL_mean": -551.41357421875, + "KL/std": 224.55908203125, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0036799819208681583, + "fcm_dpo/delta": -0.4179691672325134, + "fcm_dpo/margin": 221.39480590820312, + "fcm_dpo/q_t": 0.3275856375694275, + "grad_norm": 34.93635177612305, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": 0.17583899199962616, + "logits/rejected": 0.16290965676307678, + "logps/chosen": -381.37176513671875, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -655.605224609375, + "loss": 0.8747, + "margin_dpo/margin_mean": 221.3948211669922, + "margin_dpo/margin_std": 222.51441955566406, + "step": 380 + }, + { + "KL/chosen_KL_mean": -346.26275634765625, + "KL/mean": -437.649169921875, + "KL/rejected_KL_mean": -529.0355834960938, + "KL/std": 188.0095672607422, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0036549135111272335, + "fcm_dpo/delta": -0.2698676884174347, + "fcm_dpo/margin": 182.77279663085938, + "fcm_dpo/q_t": 0.36184006929397583, + "grad_norm": 46.053001403808594, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": 0.20569837093353271, + "logits/rejected": 0.2217743843793869, + "logps/chosen": -404.0658264160156, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -608.2550048828125, + "loss": 0.9843, + "margin_dpo/margin_mean": 182.77279663085938, + "margin_dpo/margin_std": 238.05979919433594, + "step": 381 + }, + { + "KL/chosen_KL_mean": -348.89215087890625, + "KL/mean": -436.17047119140625, + "KL/rejected_KL_mean": -523.4487915039062, + "KL/std": 186.67771911621094, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0036372016184031963, + "fcm_dpo/delta": -0.23642855882644653, + "fcm_dpo/margin": 174.55662536621094, + "fcm_dpo/q_t": 0.36145347356796265, + "grad_norm": 43.260498046875, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": 0.13472796976566315, + "logits/rejected": 0.13556230068206787, + "logps/chosen": -414.9124755859375, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -634.158935546875, + "loss": 0.9534, + "margin_dpo/margin_mean": 174.556640625, + "margin_dpo/margin_std": 189.3209228515625, + "step": 382 + }, + { + "KL/chosen_KL_mean": -324.9818115234375, + "KL/mean": -420.0364990234375, + "KL/rejected_KL_mean": -515.0911254882812, + "KL/std": 197.85250854492188, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.003616844303905964, + "fcm_dpo/delta": -0.2896538972854614, + "fcm_dpo/margin": 190.10928344726562, + "fcm_dpo/q_t": 0.3557220697402954, + "grad_norm": 52.62465286254883, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": 0.2032158523797989, + "logits/rejected": 0.19827163219451904, + "logps/chosen": -375.373291015625, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -608.8070068359375, + "loss": 0.9764, + "margin_dpo/margin_mean": 190.10928344726562, + "margin_dpo/margin_std": 243.69564819335938, + "step": 383 + }, + { + "KL/chosen_KL_mean": -306.2908935546875, + "KL/mean": -385.357666015625, + "KL/rejected_KL_mean": -464.42431640625, + "KL/std": 184.37274169921875, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0036061767023056746, + "fcm_dpo/delta": -0.17123505473136902, + "fcm_dpo/margin": 158.13345336914062, + "fcm_dpo/q_t": 0.37064245343208313, + "grad_norm": 32.06476974487305, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": 0.03205307945609093, + "logits/rejected": 0.03471808135509491, + "logps/chosen": -358.3370056152344, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -550.1852416992188, + "loss": 0.9718, + "margin_dpo/margin_mean": 158.13345336914062, + "margin_dpo/margin_std": 154.05892944335938, + "step": 384 + }, + { + "KL/chosen_KL_mean": -277.21063232421875, + "KL/mean": -372.15472412109375, + "KL/rejected_KL_mean": -467.0987548828125, + "KL/std": 164.99423217773438, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0035849306732416153, + "fcm_dpo/delta": -0.2826468348503113, + "fcm_dpo/margin": 189.88804626464844, + "fcm_dpo/q_t": 0.3470049798488617, + "grad_norm": 40.691505432128906, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": 0.1265610307455063, + "logits/rejected": 0.16434606909751892, + "logps/chosen": -342.7628173828125, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -544.9266357421875, + "loss": 0.8934, + "margin_dpo/margin_mean": 189.88804626464844, + "margin_dpo/margin_std": 162.15029907226562, + "step": 385 + }, + { + "KL/chosen_KL_mean": -283.49786376953125, + "KL/mean": -381.19476318359375, + "KL/rejected_KL_mean": -478.8916015625, + "KL/std": 188.57232666015625, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.003565252758562565, + "fcm_dpo/delta": -0.2986172139644623, + "fcm_dpo/margin": 195.39376831054688, + "fcm_dpo/q_t": 0.35123616456985474, + "grad_norm": 37.065982818603516, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": 0.06240156292915344, + "logits/rejected": 0.07301077246665955, + "logps/chosen": -341.7197265625, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -571.218994140625, + "loss": 0.9247, + "margin_dpo/margin_mean": 195.39376831054688, + "margin_dpo/margin_std": 209.2765350341797, + "step": 386 + }, + { + "KL/chosen_KL_mean": -295.6245422363281, + "KL/mean": -370.982421875, + "KL/rejected_KL_mean": -446.3402404785156, + "KL/std": 165.49749755859375, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.003549664979800582, + "fcm_dpo/delta": -0.13580653071403503, + "fcm_dpo/margin": 150.71572875976562, + "fcm_dpo/q_t": 0.3772751986980438, + "grad_norm": 37.2304573059082, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": 0.03751121088862419, + "logits/rejected": 0.065042644739151, + "logps/chosen": -362.04400634765625, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -538.5093994140625, + "loss": 0.9867, + "margin_dpo/margin_mean": 150.7157440185547, + "margin_dpo/margin_std": 149.2532958984375, + "step": 387 + }, + { + "KL/chosen_KL_mean": -294.2076110839844, + "KL/mean": -395.94500732421875, + "KL/rejected_KL_mean": -497.6824951171875, + "KL/std": 186.8450927734375, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0035306853242218494, + "fcm_dpo/delta": -0.32081812620162964, + "fcm_dpo/margin": 203.47482299804688, + "fcm_dpo/q_t": 0.34532350301742554, + "grad_norm": 52.664127349853516, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": 0.10607756674289703, + "logits/rejected": 0.09465403854846954, + "logps/chosen": -344.3370666503906, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -602.1155395507812, + "loss": 0.9005, + "margin_dpo/margin_mean": 203.47482299804688, + "margin_dpo/margin_std": 203.6334686279297, + "step": 388 + }, + { + "KL/chosen_KL_mean": -305.42205810546875, + "KL/mean": -403.931640625, + "KL/rejected_KL_mean": -502.441162109375, + "KL/std": 193.5303955078125, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.0035098318476229906, + "fcm_dpo/delta": -0.29360121488571167, + "fcm_dpo/margin": 197.01910400390625, + "fcm_dpo/q_t": 0.35070139169692993, + "grad_norm": 34.60560607910156, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": 0.16850537061691284, + "logits/rejected": 0.19408723711967468, + "logps/chosen": -363.32867431640625, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -580.355712890625, + "loss": 0.9207, + "margin_dpo/margin_mean": 197.01910400390625, + "margin_dpo/margin_std": 203.4574737548828, + "step": 389 + }, + { + "KL/chosen_KL_mean": -317.38275146484375, + "KL/mean": -402.5523681640625, + "KL/rejected_KL_mean": -487.7220458984375, + "KL/std": 181.0073699951172, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.0034929700195789337, + "fcm_dpo/delta": -0.19634784758090973, + "fcm_dpo/margin": 170.3392791748047, + "fcm_dpo/q_t": 0.37371453642845154, + "grad_norm": 53.93910598754883, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": 0.12302356958389282, + "logits/rejected": 0.13908526301383972, + "logps/chosen": -366.6086730957031, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -573.250244140625, + "loss": 1.0133, + "margin_dpo/margin_mean": 170.33929443359375, + "margin_dpo/margin_std": 223.96063232421875, + "step": 390 + }, + { + "KL/chosen_KL_mean": -311.1605529785156, + "KL/mean": -369.17108154296875, + "KL/rejected_KL_mean": -427.1815490722656, + "KL/std": 180.6544952392578, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0034915052819997072, + "fcm_dpo/delta": -0.0051087187603116035, + "fcm_dpo/margin": 116.02098846435547, + "fcm_dpo/q_t": 0.4115014672279358, + "grad_norm": 51.79216003417969, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": 0.10646455734968185, + "logits/rejected": 0.11065554618835449, + "logps/chosen": -375.4902038574219, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -513.9197387695312, + "loss": 1.118, + "margin_dpo/margin_mean": 116.02098846435547, + "margin_dpo/margin_std": 186.80828857421875, + "step": 391 + }, + { + "KL/chosen_KL_mean": -272.8625183105469, + "KL/mean": -373.8502197265625, + "KL/rejected_KL_mean": -474.8379211425781, + "KL/std": 188.267822265625, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0034768860787153244, + "fcm_dpo/delta": -0.30428948998451233, + "fcm_dpo/margin": 201.97535705566406, + "fcm_dpo/q_t": 0.34588244557380676, + "grad_norm": 37.3916015625, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": 0.07356823980808258, + "logits/rejected": 0.08689874410629272, + "logps/chosen": -326.36651611328125, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -577.1837158203125, + "loss": 0.8887, + "margin_dpo/margin_mean": 201.97537231445312, + "margin_dpo/margin_std": 180.137939453125, + "step": 392 + }, + { + "KL/chosen_KL_mean": -283.85150146484375, + "KL/mean": -362.49029541015625, + "KL/rejected_KL_mean": -441.129150390625, + "KL/std": 192.00424194335938, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0034616896882653236, + "fcm_dpo/delta": -0.14528536796569824, + "fcm_dpo/margin": 157.27764892578125, + "fcm_dpo/q_t": 0.38673439621925354, + "grad_norm": 42.511112213134766, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": 0.15366870164871216, + "logits/rejected": 0.15913140773773193, + "logps/chosen": -330.3254089355469, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -513.0980224609375, + "loss": 1.0451, + "margin_dpo/margin_mean": 157.27764892578125, + "margin_dpo/margin_std": 229.56396484375, + "step": 393 + }, + { + "KL/chosen_KL_mean": -297.82940673828125, + "KL/mean": -388.8878173828125, + "KL/rejected_KL_mean": -479.9462585449219, + "KL/std": 197.00352478027344, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.003447656985372305, + "fcm_dpo/delta": -0.22935348749160767, + "fcm_dpo/margin": 182.1168212890625, + "fcm_dpo/q_t": 0.36579516530036926, + "grad_norm": 37.86737060546875, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": 0.0921085998415947, + "logits/rejected": 0.0971919596195221, + "logps/chosen": -350.740966796875, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -570.7689208984375, + "loss": 0.9691, + "margin_dpo/margin_mean": 182.1168212890625, + "margin_dpo/margin_std": 213.17315673828125, + "step": 394 + }, + { + "KL/chosen_KL_mean": -295.48150634765625, + "KL/mean": -394.4334411621094, + "KL/rejected_KL_mean": -493.3853759765625, + "KL/std": 201.99176025390625, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0034283739514648914, + "fcm_dpo/delta": -0.28053510189056396, + "fcm_dpo/margin": 197.90386962890625, + "fcm_dpo/q_t": 0.3562726676464081, + "grad_norm": 39.62918472290039, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": 0.14519137144088745, + "logits/rejected": 0.15589168667793274, + "logps/chosen": -358.02764892578125, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -577.16796875, + "loss": 0.948, + "margin_dpo/margin_mean": 197.90386962890625, + "margin_dpo/margin_std": 226.52955627441406, + "step": 395 + }, + { + "KL/chosen_KL_mean": -307.2212829589844, + "KL/mean": -396.8277587890625, + "KL/rejected_KL_mean": -486.43426513671875, + "KL/std": 192.98794555664062, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.003413173370063305, + "fcm_dpo/delta": -0.21305152773857117, + "fcm_dpo/margin": 179.2130126953125, + "fcm_dpo/q_t": 0.36398619413375854, + "grad_norm": 49.91170120239258, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": 0.08772768080234528, + "logits/rejected": 0.11062973737716675, + "logps/chosen": -376.21722412109375, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -575.0809326171875, + "loss": 0.9467, + "margin_dpo/margin_mean": 179.21299743652344, + "margin_dpo/margin_std": 180.69955444335938, + "step": 396 + }, + { + "KL/chosen_KL_mean": -302.2781677246094, + "KL/mean": -409.4823303222656, + "KL/rejected_KL_mean": -516.6865234375, + "KL/std": 193.16355895996094, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0033913846127688885, + "fcm_dpo/delta": -0.32982921600341797, + "fcm_dpo/margin": 214.40834045410156, + "fcm_dpo/q_t": 0.3414739966392517, + "grad_norm": 43.24444580078125, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": 0.17958784103393555, + "logits/rejected": 0.18757876753807068, + "logps/chosen": -363.5553283691406, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -619.8026123046875, + "loss": 0.8831, + "margin_dpo/margin_mean": 214.40834045410156, + "margin_dpo/margin_std": 195.48153686523438, + "step": 397 + }, + { + "KL/chosen_KL_mean": -335.28326416015625, + "KL/mean": -435.4043273925781, + "KL/rejected_KL_mean": -535.525390625, + "KL/std": 207.45628356933594, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0033732731826603413, + "fcm_dpo/delta": -0.2774016261100769, + "fcm_dpo/margin": 200.24215698242188, + "fcm_dpo/q_t": 0.35684192180633545, + "grad_norm": 33.972076416015625, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": 0.024152668192982674, + "logits/rejected": 0.03423169255256653, + "logps/chosen": -403.434814453125, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -644.0490112304688, + "loss": 0.9427, + "margin_dpo/margin_mean": 200.24215698242188, + "margin_dpo/margin_std": 226.54159545898438, + "step": 398 + }, + { + "KL/chosen_KL_mean": -309.7159118652344, + "KL/mean": -405.8687744140625, + "KL/rejected_KL_mean": -502.0215759277344, + "KL/std": 190.2009735107422, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0033562048338353634, + "fcm_dpo/delta": -0.24705612659454346, + "fcm_dpo/margin": 192.3056640625, + "fcm_dpo/q_t": 0.3609922528266907, + "grad_norm": 36.394107818603516, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": 0.1804802417755127, + "logits/rejected": 0.2315608561038971, + "logps/chosen": -370.605712890625, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -579.9871826171875, + "loss": 0.9613, + "margin_dpo/margin_mean": 192.3056640625, + "margin_dpo/margin_std": 220.65533447265625, + "step": 399 + }, + { + "KL/chosen_KL_mean": -320.55322265625, + "KL/mean": -452.35040283203125, + "KL/rejected_KL_mean": -584.1475219726562, + "KL/std": 211.8714141845703, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0033267298713326454, + "fcm_dpo/delta": -0.4814392030239105, + "fcm_dpo/margin": 263.5943298339844, + "fcm_dpo/q_t": 0.31366732716560364, + "grad_norm": 31.81171417236328, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": 0.18651407957077026, + "logits/rejected": 0.20100629329681396, + "logps/chosen": -384.1968078613281, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -689.4002075195312, + "loss": 0.81, + "margin_dpo/margin_mean": 263.5943298339844, + "margin_dpo/margin_std": 226.67686462402344, + "step": 400 + }, + { + "KL/chosen_KL_mean": -362.877685546875, + "KL/mean": -440.9382629394531, + "KL/rejected_KL_mean": -518.998779296875, + "KL/std": 206.93365478515625, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0033134697005152702, + "fcm_dpo/delta": -0.11797553300857544, + "fcm_dpo/margin": 156.12115478515625, + "fcm_dpo/q_t": 0.3939567804336548, + "grad_norm": 46.97633743286133, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": 0.22994808852672577, + "logits/rejected": 0.23952946066856384, + "logps/chosen": -420.04071044921875, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -602.7913208007812, + "loss": 1.1079, + "margin_dpo/margin_mean": 156.1211395263672, + "margin_dpo/margin_std": 270.3221435546875, + "step": 401 + }, + { + "KL/chosen_KL_mean": -305.3973083496094, + "KL/mean": -440.08123779296875, + "KL/rejected_KL_mean": -574.76513671875, + "KL/std": 228.32980346679688, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0032884571701288223, + "fcm_dpo/delta": -0.4901397228240967, + "fcm_dpo/margin": 269.36785888671875, + "fcm_dpo/q_t": 0.31113818287849426, + "grad_norm": 30.754236221313477, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": 0.1997782588005066, + "logits/rejected": 0.2184348702430725, + "logps/chosen": -356.1376953125, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -655.8112182617188, + "loss": 0.7859, + "margin_dpo/margin_mean": 269.36785888671875, + "margin_dpo/margin_std": 214.45001220703125, + "step": 402 + }, + { + "KL/chosen_KL_mean": -348.4799499511719, + "KL/mean": -451.6943054199219, + "KL/rejected_KL_mean": -554.90869140625, + "KL/std": 201.2946014404297, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0032661817967891693, + "fcm_dpo/delta": -0.27615201473236084, + "fcm_dpo/margin": 206.42874145507812, + "fcm_dpo/q_t": 0.35180985927581787, + "grad_norm": 43.93153381347656, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": 0.2825523614883423, + "logits/rejected": 0.2911122441291809, + "logps/chosen": -395.62725830078125, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -632.17529296875, + "loss": 0.9161, + "margin_dpo/margin_mean": 206.42874145507812, + "margin_dpo/margin_std": 198.97940063476562, + "step": 403 + }, + { + "KL/chosen_KL_mean": -363.72509765625, + "KL/mean": -468.47967529296875, + "KL/rejected_KL_mean": -573.2342529296875, + "KL/std": 193.71627807617188, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0032488093711435795, + "fcm_dpo/delta": -0.28255146741867065, + "fcm_dpo/margin": 209.50914001464844, + "fcm_dpo/q_t": 0.35515096783638, + "grad_norm": 48.310813903808594, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": 0.28808605670928955, + "logits/rejected": 0.30502545833587646, + "logps/chosen": -411.6003723144531, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -650.3892211914062, + "loss": 0.9524, + "margin_dpo/margin_mean": 209.50914001464844, + "margin_dpo/margin_std": 242.30758666992188, + "step": 404 + }, + { + "KL/chosen_KL_mean": -388.3827209472656, + "KL/mean": -474.906982421875, + "KL/rejected_KL_mean": -561.4312744140625, + "KL/std": 205.4340362548828, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.003239218145608902, + "fcm_dpo/delta": -0.16148240864276886, + "fcm_dpo/margin": 173.04855346679688, + "fcm_dpo/q_t": 0.38057857751846313, + "grad_norm": 62.34660720825195, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": 0.21315240859985352, + "logits/rejected": 0.23247873783111572, + "logps/chosen": -453.5456237792969, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -648.6180419921875, + "loss": 1.0291, + "margin_dpo/margin_mean": 173.04852294921875, + "margin_dpo/margin_std": 227.03570556640625, + "step": 405 + }, + { + "KL/chosen_KL_mean": -371.4407958984375, + "KL/mean": -474.3310546875, + "KL/rejected_KL_mean": -577.2212524414062, + "KL/std": 217.42138671875, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0032229560893028975, + "fcm_dpo/delta": -0.26489248871803284, + "fcm_dpo/margin": 205.78050231933594, + "fcm_dpo/q_t": 0.35921865701675415, + "grad_norm": 49.60548400878906, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": 0.2650710940361023, + "logits/rejected": 0.29096391797065735, + "logps/chosen": -421.18157958984375, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -669.2999267578125, + "loss": 0.9484, + "margin_dpo/margin_mean": 205.78048706054688, + "margin_dpo/margin_std": 230.2679443359375, + "step": 406 + }, + { + "KL/chosen_KL_mean": -392.8271789550781, + "KL/mean": -471.32684326171875, + "KL/rejected_KL_mean": -549.8264770507812, + "KL/std": 178.65635681152344, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.003213329706341028, + "fcm_dpo/delta": -0.10503564774990082, + "fcm_dpo/margin": 156.99928283691406, + "fcm_dpo/q_t": 0.38629892468452454, + "grad_norm": 46.0823974609375, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": 0.2919875681400299, + "logits/rejected": 0.32223090529441833, + "logps/chosen": -449.15789794921875, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -627.3385620117188, + "loss": 1.0526, + "margin_dpo/margin_mean": 156.99928283691406, + "margin_dpo/margin_std": 208.95578002929688, + "step": 407 + }, + { + "KL/chosen_KL_mean": -375.9964599609375, + "KL/mean": -452.76800537109375, + "KL/rejected_KL_mean": -529.53955078125, + "KL/std": 163.33570861816406, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.003204584354534745, + "fcm_dpo/delta": -0.09251818805932999, + "fcm_dpo/margin": 153.5430908203125, + "fcm_dpo/q_t": 0.38823240995407104, + "grad_norm": 34.644065856933594, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": 0.23389890789985657, + "logits/rejected": 0.2698257565498352, + "logps/chosen": -445.7857666015625, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -619.636474609375, + "loss": 1.029, + "margin_dpo/margin_mean": 153.54307556152344, + "margin_dpo/margin_std": 179.04273986816406, + "step": 408 + }, + { + "KL/chosen_KL_mean": -363.87567138671875, + "KL/mean": -445.357177734375, + "KL/rejected_KL_mean": -526.838623046875, + "KL/std": 197.6404571533203, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0031984669622033834, + "fcm_dpo/delta": -0.12184996902942657, + "fcm_dpo/margin": 162.96292114257812, + "fcm_dpo/q_t": 0.38322368264198303, + "grad_norm": 63.751319885253906, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": 0.14486238360404968, + "logits/rejected": 0.16435211896896362, + "logps/chosen": -431.193115234375, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -611.742919921875, + "loss": 1.0091, + "margin_dpo/margin_mean": 162.96292114257812, + "margin_dpo/margin_std": 183.711669921875, + "step": 409 + }, + { + "KL/chosen_KL_mean": -356.5075378417969, + "KL/mean": -444.84844970703125, + "KL/rejected_KL_mean": -533.1893920898438, + "KL/std": 191.71371459960938, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0031875791028141975, + "fcm_dpo/delta": -0.16413608193397522, + "fcm_dpo/margin": 176.68182373046875, + "fcm_dpo/q_t": 0.372356116771698, + "grad_norm": 34.124786376953125, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.1958819329738617, + "logits/rejected": 0.20853480696678162, + "logps/chosen": -407.972900390625, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -616.3883666992188, + "loss": 0.9807, + "margin_dpo/margin_mean": 176.68182373046875, + "margin_dpo/margin_std": 186.82904052734375, + "step": 410 + }, + { + "KL/chosen_KL_mean": -375.66473388671875, + "KL/mean": -469.17132568359375, + "KL/rejected_KL_mean": -562.6778564453125, + "KL/std": 200.4876708984375, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.003172741737216711, + "fcm_dpo/delta": -0.19492484629154205, + "fcm_dpo/margin": 187.01324462890625, + "fcm_dpo/q_t": 0.37155526876449585, + "grad_norm": 47.495994567871094, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": 0.3162494897842407, + "logits/rejected": 0.33067888021469116, + "logps/chosen": -427.97198486328125, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -643.3728637695312, + "loss": 0.9814, + "margin_dpo/margin_mean": 187.01324462890625, + "margin_dpo/margin_std": 210.31777954101562, + "step": 411 + }, + { + "KL/chosen_KL_mean": -377.25396728515625, + "KL/mean": -469.9276123046875, + "KL/rejected_KL_mean": -562.601318359375, + "KL/std": 185.7986297607422, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0031640194356441498, + "fcm_dpo/delta": -0.1875452995300293, + "fcm_dpo/margin": 185.3472900390625, + "fcm_dpo/q_t": 0.37129780650138855, + "grad_norm": 34.143882751464844, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": 0.2625766694545746, + "logits/rejected": 0.25507473945617676, + "logps/chosen": -430.3981018066406, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -662.662109375, + "loss": 0.9882, + "margin_dpo/margin_mean": 185.34732055664062, + "margin_dpo/margin_std": 216.66151428222656, + "step": 412 + }, + { + "KL/chosen_KL_mean": -380.51409912109375, + "KL/mean": -473.68408203125, + "KL/rejected_KL_mean": -566.8541259765625, + "KL/std": 197.8855743408203, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.003152801189571619, + "fcm_dpo/delta": -0.18857158720493317, + "fcm_dpo/margin": 186.34002685546875, + "fcm_dpo/q_t": 0.36703750491142273, + "grad_norm": 32.571876525878906, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": 0.17814315855503082, + "logits/rejected": 0.19190964102745056, + "logps/chosen": -442.0960693359375, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -666.3275146484375, + "loss": 0.9657, + "margin_dpo/margin_mean": 186.34002685546875, + "margin_dpo/margin_std": 191.57870483398438, + "step": 413 + }, + { + "KL/chosen_KL_mean": -315.71142578125, + "KL/mean": -428.3543701171875, + "KL/rejected_KL_mean": -540.997314453125, + "KL/std": 188.96395874023438, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0031369260977953672, + "fcm_dpo/delta": -0.30876976251602173, + "fcm_dpo/margin": 225.28585815429688, + "fcm_dpo/q_t": 0.34278273582458496, + "grad_norm": 32.67140579223633, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": 0.2569258213043213, + "logits/rejected": 0.2717755436897278, + "logps/chosen": -362.3428955078125, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -628.643798828125, + "loss": 0.8829, + "margin_dpo/margin_mean": 225.28585815429688, + "margin_dpo/margin_std": 189.728759765625, + "step": 414 + }, + { + "KL/chosen_KL_mean": -343.6112060546875, + "KL/mean": -427.80975341796875, + "KL/rejected_KL_mean": -512.00830078125, + "KL/std": 194.63278198242188, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0031224254053086042, + "fcm_dpo/delta": -0.12657414376735687, + "fcm_dpo/margin": 168.39703369140625, + "fcm_dpo/q_t": 0.38202571868896484, + "grad_norm": 37.11186218261719, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": 0.13525229692459106, + "logits/rejected": 0.14854013919830322, + "logps/chosen": -422.2294921875, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -612.4857788085938, + "loss": 1.016, + "margin_dpo/margin_mean": 168.39703369140625, + "margin_dpo/margin_std": 197.4344482421875, + "step": 415 + }, + { + "KL/chosen_KL_mean": -306.2261962890625, + "KL/mean": -415.98931884765625, + "KL/rejected_KL_mean": -525.7525024414062, + "KL/std": 198.09100341796875, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.0031101000495254993, + "fcm_dpo/delta": -0.28460225462913513, + "fcm_dpo/margin": 219.52627563476562, + "fcm_dpo/q_t": 0.3465028405189514, + "grad_norm": 48.52604675292969, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": 0.16062219440937042, + "logits/rejected": 0.1683279424905777, + "logps/chosen": -364.50531005859375, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -616.3211669921875, + "loss": 0.8906, + "margin_dpo/margin_mean": 219.52627563476562, + "margin_dpo/margin_std": 182.83580017089844, + "step": 416 + }, + { + "KL/chosen_KL_mean": -299.74609375, + "KL/mean": -402.9626159667969, + "KL/rejected_KL_mean": -506.17913818359375, + "KL/std": 187.6376495361328, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.003093605861067772, + "fcm_dpo/delta": -0.2401183694601059, + "fcm_dpo/margin": 206.43304443359375, + "fcm_dpo/q_t": 0.3549308180809021, + "grad_norm": 46.05820083618164, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": 0.3069426417350769, + "logits/rejected": 0.3386053442955017, + "logps/chosen": -349.94482421875, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -574.3309936523438, + "loss": 0.9101, + "margin_dpo/margin_mean": 206.4330596923828, + "margin_dpo/margin_std": 172.1961669921875, + "step": 417 + }, + { + "KL/chosen_KL_mean": -313.6793212890625, + "KL/mean": -409.97967529296875, + "KL/rejected_KL_mean": -506.27996826171875, + "KL/std": 208.83714294433594, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.003082898911088705, + "fcm_dpo/delta": -0.19490408897399902, + "fcm_dpo/margin": 192.60064697265625, + "fcm_dpo/q_t": 0.3709861636161804, + "grad_norm": 32.75982666015625, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": 0.11463148891925812, + "logits/rejected": 0.14268271625041962, + "logps/chosen": -395.65777587890625, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -610.971435546875, + "loss": 0.984, + "margin_dpo/margin_mean": 192.60064697265625, + "margin_dpo/margin_std": 219.88079833984375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -283.20751953125, + "KL/mean": -387.401123046875, + "KL/rejected_KL_mean": -491.5947265625, + "KL/std": 188.24075317382812, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0030659371986985207, + "fcm_dpo/delta": -0.24046191573143005, + "fcm_dpo/margin": 208.38720703125, + "fcm_dpo/q_t": 0.3579530715942383, + "grad_norm": 26.5821533203125, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": 0.06853337585926056, + "logits/rejected": 0.07725735008716583, + "logps/chosen": -336.15618896484375, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -583.1778564453125, + "loss": 0.9334, + "margin_dpo/margin_mean": 208.38720703125, + "margin_dpo/margin_std": 203.7388916015625, + "step": 419 + }, + { + "KL/chosen_KL_mean": -316.89324951171875, + "KL/mean": -396.66534423828125, + "KL/rejected_KL_mean": -476.4374694824219, + "KL/std": 186.04684448242188, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.003059042850509286, + "fcm_dpo/delta": -0.08847498893737793, + "fcm_dpo/margin": 159.54425048828125, + "fcm_dpo/q_t": 0.393548846244812, + "grad_norm": 53.08361053466797, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": 0.17812876403331757, + "logits/rejected": 0.22114044427871704, + "logps/chosen": -394.66314697265625, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -545.75732421875, + "loss": 1.0602, + "margin_dpo/margin_mean": 159.54425048828125, + "margin_dpo/margin_std": 225.21783447265625, + "step": 420 + }, + { + "KL/chosen_KL_mean": -279.3998718261719, + "KL/mean": -359.7358703613281, + "KL/rejected_KL_mean": -440.0718688964844, + "KL/std": 177.16258239746094, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0030536255799233913, + "fcm_dpo/delta": -0.09106864035129547, + "fcm_dpo/margin": 160.6719970703125, + "fcm_dpo/q_t": 0.3875048756599426, + "grad_norm": 57.91465759277344, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": 0.10739485919475555, + "logits/rejected": 0.12462041527032852, + "logps/chosen": -333.1657409667969, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -529.3533325195312, + "loss": 1.0315, + "margin_dpo/margin_mean": 160.6719970703125, + "margin_dpo/margin_std": 186.3178253173828, + "step": 421 + }, + { + "KL/chosen_KL_mean": -318.76470947265625, + "KL/mean": -404.734619140625, + "KL/rejected_KL_mean": -490.7044677734375, + "KL/std": 193.9627685546875, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.003046384546905756, + "fcm_dpo/delta": -0.12443090975284576, + "fcm_dpo/margin": 171.93975830078125, + "fcm_dpo/q_t": 0.3814762830734253, + "grad_norm": 40.762638092041016, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": 0.20998799800872803, + "logits/rejected": 0.2345445454120636, + "logps/chosen": -387.39849853515625, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -578.5679931640625, + "loss": 1.023, + "margin_dpo/margin_mean": 171.93975830078125, + "margin_dpo/margin_std": 209.70448303222656, + "step": 422 + }, + { + "KL/chosen_KL_mean": -322.2258605957031, + "KL/mean": -414.66583251953125, + "KL/rejected_KL_mean": -507.1058349609375, + "KL/std": 169.60302734375, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0030361097306013107, + "fcm_dpo/delta": -0.16225537657737732, + "fcm_dpo/margin": 184.87997436523438, + "fcm_dpo/q_t": 0.37296515703201294, + "grad_norm": 43.99137496948242, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": 0.20146864652633667, + "logits/rejected": 0.20896153151988983, + "logps/chosen": -377.2161560058594, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -593.4124145507812, + "loss": 0.9757, + "margin_dpo/margin_mean": 184.87997436523438, + "margin_dpo/margin_std": 188.61050415039062, + "step": 423 + }, + { + "KL/chosen_KL_mean": -300.28369140625, + "KL/mean": -395.24432373046875, + "KL/rejected_KL_mean": -490.20501708984375, + "KL/std": 191.0120849609375, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.003024652134627104, + "fcm_dpo/delta": -0.1755865514278412, + "fcm_dpo/margin": 189.9213104248047, + "fcm_dpo/q_t": 0.3748074769973755, + "grad_norm": 65.47210693359375, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": 0.12795552611351013, + "logits/rejected": 0.15698958933353424, + "logps/chosen": -356.29559326171875, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -556.6839599609375, + "loss": 0.9904, + "margin_dpo/margin_mean": 189.9213104248047, + "margin_dpo/margin_std": 220.03526306152344, + "step": 424 + }, + { + "KL/chosen_KL_mean": -339.0987548828125, + "KL/mean": -416.7357177734375, + "KL/rejected_KL_mean": -494.3726806640625, + "KL/std": 179.71466064453125, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.0030212169513106346, + "fcm_dpo/delta": -0.06943905353546143, + "fcm_dpo/margin": 155.27391052246094, + "fcm_dpo/q_t": 0.3923855125904083, + "grad_norm": 46.09679412841797, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": 0.21064117550849915, + "logits/rejected": 0.20932695269584656, + "logps/chosen": -385.9677734375, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -590.2981567382812, + "loss": 1.0565, + "margin_dpo/margin_mean": 155.27392578125, + "margin_dpo/margin_std": 198.1529541015625, + "step": 425 + }, + { + "KL/chosen_KL_mean": -313.76702880859375, + "KL/mean": -419.0831604003906, + "KL/rejected_KL_mean": -524.3992919921875, + "KL/std": 194.03909301757812, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.003010502550750971, + "fcm_dpo/delta": -0.23553408682346344, + "fcm_dpo/margin": 210.6322784423828, + "fcm_dpo/q_t": 0.3583242893218994, + "grad_norm": 32.83906555175781, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": 0.054779551923274994, + "logits/rejected": 0.10161453485488892, + "logps/chosen": -390.3505859375, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -605.6658935546875, + "loss": 0.9285, + "margin_dpo/margin_mean": 210.63226318359375, + "margin_dpo/margin_std": 193.08489990234375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -323.23187255859375, + "KL/mean": -400.771484375, + "KL/rejected_KL_mean": -478.3110656738281, + "KL/std": 164.91958618164062, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0030015779193490744, + "fcm_dpo/delta": -0.0658065527677536, + "fcm_dpo/margin": 155.07919311523438, + "fcm_dpo/q_t": 0.39380916953086853, + "grad_norm": 53.79066467285156, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": 0.11577435582876205, + "logits/rejected": 0.14346517622470856, + "logps/chosen": -388.08575439453125, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -556.8770751953125, + "loss": 1.0402, + "margin_dpo/margin_mean": 155.07919311523438, + "margin_dpo/margin_std": 178.50650024414062, + "step": 427 + }, + { + "KL/chosen_KL_mean": -385.3138732910156, + "KL/mean": -478.238525390625, + "KL/rejected_KL_mean": -571.1632080078125, + "KL/std": 200.2725830078125, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.00299345375970006, + "fcm_dpo/delta": -0.15724584460258484, + "fcm_dpo/margin": 185.8493194580078, + "fcm_dpo/q_t": 0.37622326612472534, + "grad_norm": 44.164527893066406, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": 0.07598091661930084, + "logits/rejected": 0.08938292413949966, + "logps/chosen": -447.9505310058594, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -674.4449462890625, + "loss": 1.0088, + "margin_dpo/margin_mean": 185.84933471679688, + "margin_dpo/margin_std": 225.4869384765625, + "step": 428 + }, + { + "KL/chosen_KL_mean": -389.8892822265625, + "KL/mean": -478.85894775390625, + "KL/rejected_KL_mean": -567.8286743164062, + "KL/std": 179.87161254882812, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0029879738576710224, + "fcm_dpo/delta": -0.132393479347229, + "fcm_dpo/margin": 177.93936157226562, + "fcm_dpo/q_t": 0.3808261752128601, + "grad_norm": 38.39482116699219, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": 0.14956605434417725, + "logits/rejected": 0.20004940032958984, + "logps/chosen": -471.123291015625, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -659.6235961914062, + "loss": 1.015, + "margin_dpo/margin_mean": 177.93936157226562, + "margin_dpo/margin_std": 206.3253173828125, + "step": 429 + }, + { + "KL/chosen_KL_mean": -362.76312255859375, + "KL/mean": -445.71331787109375, + "KL/rejected_KL_mean": -528.6635131835938, + "KL/std": 202.63619995117188, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.002980634104460478, + "fcm_dpo/delta": -0.0949501022696495, + "fcm_dpo/margin": 165.90042114257812, + "fcm_dpo/q_t": 0.3884223699569702, + "grad_norm": 36.59517288208008, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": 0.05430976673960686, + "logits/rejected": 0.0637524351477623, + "logps/chosen": -423.6834411621094, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -633.0863037109375, + "loss": 1.0268, + "margin_dpo/margin_mean": 165.90042114257812, + "margin_dpo/margin_std": 191.83688354492188, + "step": 430 + }, + { + "KL/chosen_KL_mean": -387.6610412597656, + "KL/mean": -499.9222106933594, + "KL/rejected_KL_mean": -612.1834106445312, + "KL/std": 194.59304809570312, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0029652658849954605, + "fcm_dpo/delta": -0.26765525341033936, + "fcm_dpo/margin": 224.5224151611328, + "fcm_dpo/q_t": 0.3526460528373718, + "grad_norm": 32.7095832824707, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": 0.22222021222114563, + "logits/rejected": 0.2416287362575531, + "logps/chosen": -445.009765625, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -705.023681640625, + "loss": 0.9257, + "margin_dpo/margin_mean": 224.5224151611328, + "margin_dpo/margin_std": 219.50442504882812, + "step": 431 + }, + { + "KL/chosen_KL_mean": -374.71942138671875, + "KL/mean": -472.6753845214844, + "KL/rejected_KL_mean": -570.63134765625, + "KL/std": 204.61849975585938, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.002956526353955269, + "fcm_dpo/delta": -0.1802433431148529, + "fcm_dpo/margin": 195.91189575195312, + "fcm_dpo/q_t": 0.370572030544281, + "grad_norm": 46.127769470214844, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": 0.04833792522549629, + "logits/rejected": 0.0748792365193367, + "logps/chosen": -445.7942199707031, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -669.2108764648438, + "loss": 0.9657, + "margin_dpo/margin_mean": 195.91189575195312, + "margin_dpo/margin_std": 192.16043090820312, + "step": 432 + }, + { + "KL/chosen_KL_mean": -456.12738037109375, + "KL/mean": -528.4130859375, + "KL/rejected_KL_mean": -600.6988525390625, + "KL/std": 197.26815795898438, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0029500951059162617, + "fcm_dpo/delta": -0.02661784738302231, + "fcm_dpo/margin": 144.57150268554688, + "fcm_dpo/q_t": 0.40215736627578735, + "grad_norm": 45.17491149902344, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": 0.18638157844543457, + "logits/rejected": 0.20084905624389648, + "logps/chosen": -514.4005737304688, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -696.6497802734375, + "loss": 1.0953, + "margin_dpo/margin_mean": 144.5714874267578, + "margin_dpo/margin_std": 210.2695770263672, + "step": 433 + }, + { + "KL/chosen_KL_mean": -424.892333984375, + "KL/mean": -506.30615234375, + "KL/rejected_KL_mean": -587.7200317382812, + "KL/std": 190.89349365234375, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0029457963537424803, + "fcm_dpo/delta": -0.08006924390792847, + "fcm_dpo/margin": 162.8276824951172, + "fcm_dpo/q_t": 0.3969116806983948, + "grad_norm": 42.9658088684082, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": 0.2312847524881363, + "logits/rejected": 0.24906104803085327, + "logps/chosen": -486.86602783203125, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -666.2186279296875, + "loss": 1.0599, + "margin_dpo/margin_mean": 162.82766723632812, + "margin_dpo/margin_std": 227.4364013671875, + "step": 434 + }, + { + "KL/chosen_KL_mean": -392.6776123046875, + "KL/mean": -490.40625, + "KL/rejected_KL_mean": -588.1348876953125, + "KL/std": 201.8612060546875, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0029370298143476248, + "fcm_dpo/delta": -0.17510701715946198, + "fcm_dpo/margin": 195.4573211669922, + "fcm_dpo/q_t": 0.37184518575668335, + "grad_norm": 37.98408508300781, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": 0.342867910861969, + "logits/rejected": 0.3715318441390991, + "logps/chosen": -444.1796569824219, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -675.7017822265625, + "loss": 0.9848, + "margin_dpo/margin_mean": 195.45730590820312, + "margin_dpo/margin_std": 213.81292724609375, + "step": 435 + }, + { + "KL/chosen_KL_mean": -379.94207763671875, + "KL/mean": -477.734619140625, + "KL/rejected_KL_mean": -575.5272216796875, + "KL/std": 207.04159545898438, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0029268027283251286, + "fcm_dpo/delta": -0.17347224056720734, + "fcm_dpo/margin": 195.5851287841797, + "fcm_dpo/q_t": 0.3739010691642761, + "grad_norm": 61.16938781738281, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": 0.07705369591712952, + "logits/rejected": 0.10864249616861343, + "logps/chosen": -451.34576416015625, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -658.2550048828125, + "loss": 0.9833, + "margin_dpo/margin_mean": 195.58514404296875, + "margin_dpo/margin_std": 218.67153930664062, + "step": 436 + }, + { + "KL/chosen_KL_mean": -373.31927490234375, + "KL/mean": -464.244384765625, + "KL/rejected_KL_mean": -555.1694946289062, + "KL/std": 181.54849243164062, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0029189912602305412, + "fcm_dpo/delta": -0.1315203756093979, + "fcm_dpo/margin": 181.85023498535156, + "fcm_dpo/q_t": 0.3811717629432678, + "grad_norm": 41.396732330322266, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": 0.1856456696987152, + "logits/rejected": 0.22200891375541687, + "logps/chosen": -438.06353759765625, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -637.2130126953125, + "loss": 1.007, + "margin_dpo/margin_mean": 181.8502197265625, + "margin_dpo/margin_std": 207.84622192382812, + "step": 437 + }, + { + "KL/chosen_KL_mean": -338.4395751953125, + "KL/mean": -444.72418212890625, + "KL/rejected_KL_mean": -551.0087890625, + "KL/std": 187.68255615234375, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0029061120003461838, + "fcm_dpo/delta": -0.21926280856132507, + "fcm_dpo/margin": 212.56918334960938, + "fcm_dpo/q_t": 0.360361248254776, + "grad_norm": 56.88426208496094, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": 0.23124368488788605, + "logits/rejected": 0.274699866771698, + "logps/chosen": -397.458251953125, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -634.0855712890625, + "loss": 0.9305, + "margin_dpo/margin_mean": 212.5691680908203, + "margin_dpo/margin_std": 188.52572631835938, + "step": 438 + }, + { + "KL/chosen_KL_mean": -339.2711181640625, + "KL/mean": -422.31060791015625, + "KL/rejected_KL_mean": -505.35009765625, + "KL/std": 196.2166290283203, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0028977172914892435, + "fcm_dpo/delta": -0.08195704966783524, + "fcm_dpo/margin": 166.07899475097656, + "fcm_dpo/q_t": 0.3928077220916748, + "grad_norm": 39.39056396484375, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": 0.19246186316013336, + "logits/rejected": 0.1972828507423401, + "logps/chosen": -393.0552062988281, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -589.3355712890625, + "loss": 1.0434, + "margin_dpo/margin_mean": 166.07899475097656, + "margin_dpo/margin_std": 194.10421752929688, + "step": 439 + }, + { + "KL/chosen_KL_mean": -322.7694396972656, + "KL/mean": -402.7139587402344, + "KL/rejected_KL_mean": -482.658447265625, + "KL/std": 213.01551818847656, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0028976453468203545, + "fcm_dpo/delta": -0.06359387934207916, + "fcm_dpo/margin": 159.88897705078125, + "fcm_dpo/q_t": 0.39812254905700684, + "grad_norm": 61.76188278198242, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": 0.12061315774917603, + "logits/rejected": 0.16271516680717468, + "logps/chosen": -401.336181640625, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -579.1561889648438, + "loss": 1.0676, + "margin_dpo/margin_mean": 159.88897705078125, + "margin_dpo/margin_std": 220.2742156982422, + "step": 440 + }, + { + "KL/chosen_KL_mean": -322.7802734375, + "KL/mean": -413.9019775390625, + "KL/rejected_KL_mean": -505.023681640625, + "KL/std": 201.19439697265625, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0028908136300742626, + "fcm_dpo/delta": -0.1274890899658203, + "fcm_dpo/margin": 182.24334716796875, + "fcm_dpo/q_t": 0.38383615016937256, + "grad_norm": 36.56028366088867, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": 0.08522676676511765, + "logits/rejected": 0.10035119950771332, + "logps/chosen": -383.604736328125, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -601.4945068359375, + "loss": 1.031, + "margin_dpo/margin_mean": 182.24334716796875, + "margin_dpo/margin_std": 231.61550903320312, + "step": 441 + }, + { + "KL/chosen_KL_mean": -275.9490966796875, + "KL/mean": -385.7955322265625, + "KL/rejected_KL_mean": -495.64190673828125, + "KL/std": 204.95565795898438, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0028799688443541527, + "fcm_dpo/delta": -0.23412252962589264, + "fcm_dpo/margin": 219.69284057617188, + "fcm_dpo/q_t": 0.36125171184539795, + "grad_norm": 40.638084411621094, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": 0.07362253963947296, + "logits/rejected": 0.0884602814912796, + "logps/chosen": -322.9602966308594, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -572.18115234375, + "loss": 0.943, + "margin_dpo/margin_mean": 219.69284057617188, + "margin_dpo/margin_std": 224.29092407226562, + "step": 442 + }, + { + "KL/chosen_KL_mean": -308.8583984375, + "KL/mean": -380.9923095703125, + "KL/rejected_KL_mean": -453.1261901855469, + "KL/std": 181.59857177734375, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0028731776401400566, + "fcm_dpo/delta": -0.01461159810423851, + "fcm_dpo/margin": 144.26779174804688, + "fcm_dpo/q_t": 0.40677422285079956, + "grad_norm": 34.6506462097168, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": 0.09414487332105637, + "logits/rejected": 0.11145065724849701, + "logps/chosen": -380.1314392089844, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -539.80615234375, + "loss": 1.1044, + "margin_dpo/margin_mean": 144.26779174804688, + "margin_dpo/margin_std": 217.25619506835938, + "step": 443 + }, + { + "KL/chosen_KL_mean": -298.66680908203125, + "KL/mean": -405.908935546875, + "KL/rejected_KL_mean": -513.151123046875, + "KL/std": 206.92886352539062, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.002864152193069458, + "fcm_dpo/delta": -0.21571515500545502, + "fcm_dpo/margin": 214.48434448242188, + "fcm_dpo/q_t": 0.3648812472820282, + "grad_norm": 39.880069732666016, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.16348761320114136, + "logits/rejected": 0.1764025092124939, + "logps/chosen": -355.8804931640625, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -610.406005859375, + "loss": 0.961, + "margin_dpo/margin_mean": 214.4843292236328, + "margin_dpo/margin_std": 232.71310424804688, + "step": 444 + }, + { + "KL/chosen_KL_mean": -282.07110595703125, + "KL/mean": -386.0201416015625, + "KL/rejected_KL_mean": -489.96917724609375, + "KL/std": 183.015380859375, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.002853207290172577, + "fcm_dpo/delta": -0.1943528652191162, + "fcm_dpo/margin": 207.89810180664062, + "fcm_dpo/q_t": 0.36701154708862305, + "grad_norm": 49.25478744506836, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": 0.058859989047050476, + "logits/rejected": 0.08266326785087585, + "logps/chosen": -349.37091064453125, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -582.65185546875, + "loss": 0.9539, + "margin_dpo/margin_mean": 207.89810180664062, + "margin_dpo/margin_std": 201.21517944335938, + "step": 445 + }, + { + "KL/chosen_KL_mean": -269.1939697265625, + "KL/mean": -372.62945556640625, + "KL/rejected_KL_mean": -476.06500244140625, + "KL/std": 185.48361206054688, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.0028447716031223536, + "fcm_dpo/delta": -0.18958672881126404, + "fcm_dpo/margin": 206.8710479736328, + "fcm_dpo/q_t": 0.3665429949760437, + "grad_norm": 35.42505645751953, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": 0.1213020458817482, + "logits/rejected": 0.12550175189971924, + "logps/chosen": -328.2924499511719, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -577.3292236328125, + "loss": 0.9484, + "margin_dpo/margin_mean": 206.8710479736328, + "margin_dpo/margin_std": 183.12344360351562, + "step": 446 + }, + { + "KL/chosen_KL_mean": -285.10894775390625, + "KL/mean": -400.2750549316406, + "KL/rejected_KL_mean": -515.441162109375, + "KL/std": 226.73428344726562, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0028279973194003105, + "fcm_dpo/delta": -0.25317084789276123, + "fcm_dpo/margin": 230.3321990966797, + "fcm_dpo/q_t": 0.3578330874443054, + "grad_norm": 33.810001373291016, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": 0.1014283075928688, + "logits/rejected": 0.11137712746858597, + "logps/chosen": -341.18426513671875, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -614.1358642578125, + "loss": 0.926, + "margin_dpo/margin_mean": 230.33221435546875, + "margin_dpo/margin_std": 231.1820068359375, + "step": 447 + }, + { + "KL/chosen_KL_mean": -320.351318359375, + "KL/mean": -435.49365234375, + "KL/rejected_KL_mean": -550.635986328125, + "KL/std": 203.16171264648438, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.002816353924572468, + "fcm_dpo/delta": -0.25011274218559265, + "fcm_dpo/margin": 230.28466796875, + "fcm_dpo/q_t": 0.355141282081604, + "grad_norm": 43.4305534362793, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": 0.12146681547164917, + "logits/rejected": 0.13618314266204834, + "logps/chosen": -380.35516357421875, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -652.900634765625, + "loss": 0.913, + "margin_dpo/margin_mean": 230.28466796875, + "margin_dpo/margin_std": 203.189208984375, + "step": 448 + }, + { + "KL/chosen_KL_mean": -347.09442138671875, + "KL/mean": -450.777587890625, + "KL/rejected_KL_mean": -554.460693359375, + "KL/std": 226.61297607421875, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.0028034509159624577, + "fcm_dpo/delta": -0.18243032693862915, + "fcm_dpo/margin": 207.36630249023438, + "fcm_dpo/q_t": 0.37642303109169006, + "grad_norm": 44.95643997192383, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": 0.0925997942686081, + "logits/rejected": 0.12435147166252136, + "logps/chosen": -414.62103271484375, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -643.0576171875, + "loss": 1.019, + "margin_dpo/margin_mean": 207.36630249023438, + "margin_dpo/margin_std": 280.90875244140625, + "step": 449 + }, + { + "KL/chosen_KL_mean": -312.5721435546875, + "KL/mean": -431.3353271484375, + "KL/rejected_KL_mean": -550.0985107421875, + "KL/std": 225.60690307617188, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0027912480290979147, + "fcm_dpo/delta": -0.2646784484386444, + "fcm_dpo/margin": 237.52630615234375, + "fcm_dpo/q_t": 0.35031676292419434, + "grad_norm": 44.79782485961914, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": 0.17363858222961426, + "logits/rejected": 0.1841849684715271, + "logps/chosen": -369.6802978515625, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -652.8533935546875, + "loss": 0.9083, + "margin_dpo/margin_mean": 237.52630615234375, + "margin_dpo/margin_std": 211.3832244873047, + "step": 450 + }, + { + "KL/chosen_KL_mean": -396.2445983886719, + "KL/mean": -489.5446472167969, + "KL/rejected_KL_mean": -582.8447265625, + "KL/std": 224.094482421875, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.002782489638775587, + "fcm_dpo/delta": -0.11981803923845291, + "fcm_dpo/margin": 186.60012817382812, + "fcm_dpo/q_t": 0.3809146285057068, + "grad_norm": 55.61586380004883, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": 0.3262878656387329, + "logits/rejected": 0.36338138580322266, + "logps/chosen": -454.71343994140625, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -655.7741088867188, + "loss": 1.0468, + "margin_dpo/margin_mean": 186.60012817382812, + "margin_dpo/margin_std": 253.76104736328125, + "step": 451 + }, + { + "KL/chosen_KL_mean": -341.9442138671875, + "KL/mean": -452.24493408203125, + "KL/rejected_KL_mean": -562.5455322265625, + "KL/std": 213.12074279785156, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0027705044485628605, + "fcm_dpo/delta": -0.21251745522022247, + "fcm_dpo/margin": 220.60137939453125, + "fcm_dpo/q_t": 0.36281681060791016, + "grad_norm": 42.38181686401367, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": 0.24371537566184998, + "logits/rejected": 0.24365836381912231, + "logps/chosen": -392.33477783203125, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -660.3170166015625, + "loss": 0.9414, + "margin_dpo/margin_mean": 220.60137939453125, + "margin_dpo/margin_std": 211.88397216796875, + "step": 452 + }, + { + "KL/chosen_KL_mean": -402.0490417480469, + "KL/mean": -502.78240966796875, + "KL/rejected_KL_mean": -603.5157470703125, + "KL/std": 217.02017211914062, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0027601602487266064, + "fcm_dpo/delta": -0.15707828104496002, + "fcm_dpo/margin": 201.46673583984375, + "fcm_dpo/q_t": 0.37893903255462646, + "grad_norm": 35.151153564453125, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": 0.17022854089736938, + "logits/rejected": 0.21134544909000397, + "logps/chosen": -459.763916015625, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -685.7232055664062, + "loss": 1.0002, + "margin_dpo/margin_mean": 201.46673583984375, + "margin_dpo/margin_std": 242.64547729492188, + "step": 453 + }, + { + "KL/chosen_KL_mean": -473.6311340332031, + "KL/mean": -590.0665283203125, + "KL/rejected_KL_mean": -706.501953125, + "KL/std": 244.59690856933594, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.00274825980886817, + "fcm_dpo/delta": -0.2416648417711258, + "fcm_dpo/margin": 232.8707733154297, + "fcm_dpo/q_t": 0.3636271357536316, + "grad_norm": 38.61016845703125, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": 0.2933243215084076, + "logits/rejected": 0.31682437658309937, + "logps/chosen": -534.5767822265625, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -791.4526977539062, + "loss": 0.9645, + "margin_dpo/margin_mean": 232.87078857421875, + "margin_dpo/margin_std": 274.3940124511719, + "step": 454 + }, + { + "KL/chosen_KL_mean": -430.7701110839844, + "KL/mean": -564.3446655273438, + "KL/rejected_KL_mean": -697.919189453125, + "KL/std": 260.12060546875, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.002734083915129304, + "fcm_dpo/delta": -0.332709938287735, + "fcm_dpo/margin": 267.14910888671875, + "fcm_dpo/q_t": 0.3445233702659607, + "grad_norm": 39.126163482666016, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": 0.23422124981880188, + "logits/rejected": 0.24214929342269897, + "logps/chosen": -475.65679931640625, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -813.220703125, + "loss": 0.8964, + "margin_dpo/margin_mean": 267.14910888671875, + "margin_dpo/margin_std": 262.3526611328125, + "step": 455 + }, + { + "KL/chosen_KL_mean": -451.811279296875, + "KL/mean": -579.1569213867188, + "KL/rejected_KL_mean": -706.5025634765625, + "KL/std": 261.30426025390625, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0027158178854733706, + "fcm_dpo/delta": -0.2937299609184265, + "fcm_dpo/margin": 254.69129943847656, + "fcm_dpo/q_t": 0.3479035794734955, + "grad_norm": 41.266963958740234, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": 0.22368907928466797, + "logits/rejected": 0.22528277337551117, + "logps/chosen": -508.84808349609375, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -811.7203979492188, + "loss": 0.8907, + "margin_dpo/margin_mean": 254.69131469726562, + "margin_dpo/margin_std": 227.30397033691406, + "step": 456 + }, + { + "KL/chosen_KL_mean": -447.75201416015625, + "KL/mean": -572.9967041015625, + "KL/rejected_KL_mean": -698.241455078125, + "KL/std": 253.35531616210938, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.002699613571166992, + "fcm_dpo/delta": -0.2782280445098877, + "fcm_dpo/margin": 250.48944091796875, + "fcm_dpo/q_t": 0.34909045696258545, + "grad_norm": 42.01800537109375, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": 0.22898268699645996, + "logits/rejected": 0.269948810338974, + "logps/chosen": -501.99456787109375, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -783.35107421875, + "loss": 0.9072, + "margin_dpo/margin_mean": 250.4894256591797, + "margin_dpo/margin_std": 229.4442138671875, + "step": 457 + }, + { + "KL/chosen_KL_mean": -465.156982421875, + "KL/mean": -583.8756713867188, + "KL/rejected_KL_mean": -702.5943603515625, + "KL/std": 227.717529296875, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.002686592750251293, + "fcm_dpo/delta": -0.23948143422603607, + "fcm_dpo/margin": 237.43734741210938, + "fcm_dpo/q_t": 0.3595222532749176, + "grad_norm": 48.49515151977539, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": 0.3005247116088867, + "logits/rejected": 0.3180386424064636, + "logps/chosen": -520.5658569335938, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -800.277587890625, + "loss": 0.9359, + "margin_dpo/margin_mean": 237.43736267089844, + "margin_dpo/margin_std": 241.90618896484375, + "step": 458 + }, + { + "KL/chosen_KL_mean": -485.1322937011719, + "KL/mean": -595.32568359375, + "KL/rejected_KL_mean": -705.51904296875, + "KL/std": 228.52102661132812, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.002675633877515793, + "fcm_dpo/delta": -0.19085347652435303, + "fcm_dpo/margin": 220.38677978515625, + "fcm_dpo/q_t": 0.36975985765457153, + "grad_norm": 58.12493133544922, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": 0.2672702670097351, + "logits/rejected": 0.2801096439361572, + "logps/chosen": -531.6898193359375, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -791.6876220703125, + "loss": 0.9781, + "margin_dpo/margin_mean": 220.3867950439453, + "margin_dpo/margin_std": 242.81564331054688, + "step": 459 + }, + { + "KL/chosen_KL_mean": -508.26361083984375, + "KL/mean": -624.116455078125, + "KL/rejected_KL_mean": -739.9693603515625, + "KL/std": 234.30999755859375, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.002665161155164242, + "fcm_dpo/delta": -0.21886751055717468, + "fcm_dpo/margin": 231.70574951171875, + "fcm_dpo/q_t": 0.36404848098754883, + "grad_norm": 51.53123474121094, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": 0.20813575387001038, + "logits/rejected": 0.20157086849212646, + "logps/chosen": -559.8985595703125, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -844.0887451171875, + "loss": 0.9584, + "margin_dpo/margin_mean": 231.70574951171875, + "margin_dpo/margin_std": 250.1143798828125, + "step": 460 + }, + { + "KL/chosen_KL_mean": -495.30712890625, + "KL/mean": -590.9074096679688, + "KL/rejected_KL_mean": -686.5076904296875, + "KL/std": 236.65054321289062, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.002658518496900797, + "fcm_dpo/delta": -0.10884910821914673, + "fcm_dpo/margin": 191.2004852294922, + "fcm_dpo/q_t": 0.3879580497741699, + "grad_norm": 44.707061767578125, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": 0.1874847412109375, + "logits/rejected": 0.2009834498167038, + "logps/chosen": -550.4891357421875, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -772.9845581054688, + "loss": 1.034, + "margin_dpo/margin_mean": 191.2004852294922, + "margin_dpo/margin_std": 243.09445190429688, + "step": 461 + }, + { + "KL/chosen_KL_mean": -472.5901794433594, + "KL/mean": -560.455810546875, + "KL/rejected_KL_mean": -648.321533203125, + "KL/std": 213.39019775390625, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.002652602270245552, + "fcm_dpo/delta": -0.06650609523057938, + "fcm_dpo/margin": 175.7313232421875, + "fcm_dpo/q_t": 0.3937518000602722, + "grad_norm": 38.62584686279297, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": 0.27334287762641907, + "logits/rejected": 0.2956535816192627, + "logps/chosen": -542.5181884765625, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -727.1626586914062, + "loss": 1.0415, + "margin_dpo/margin_mean": 175.73133850097656, + "margin_dpo/margin_std": 204.4156494140625, + "step": 462 + }, + { + "KL/chosen_KL_mean": -474.4023742675781, + "KL/mean": -574.0709228515625, + "KL/rejected_KL_mean": -673.739501953125, + "KL/std": 230.12823486328125, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.002648891881108284, + "fcm_dpo/delta": -0.12868750095367432, + "fcm_dpo/margin": 199.33706665039062, + "fcm_dpo/q_t": 0.38484853506088257, + "grad_norm": 48.24089431762695, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": 0.23850375413894653, + "logits/rejected": 0.257518470287323, + "logps/chosen": -529.6767578125, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -762.764404296875, + "loss": 1.0327, + "margin_dpo/margin_mean": 199.33706665039062, + "margin_dpo/margin_std": 257.9790954589844, + "step": 463 + }, + { + "KL/chosen_KL_mean": -416.27117919921875, + "KL/mean": -561.68994140625, + "KL/rejected_KL_mean": -707.108642578125, + "KL/std": 256.990234375, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.002630078699439764, + "fcm_dpo/delta": -0.3679578900337219, + "fcm_dpo/margin": 290.83746337890625, + "fcm_dpo/q_t": 0.33440473675727844, + "grad_norm": 40.69416809082031, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": 0.23092204332351685, + "logits/rejected": 0.23742210865020752, + "logps/chosen": -467.1835021972656, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -809.5980224609375, + "loss": 0.8628, + "margin_dpo/margin_mean": 290.83746337890625, + "margin_dpo/margin_std": 262.0067138671875, + "step": 464 + }, + { + "KL/chosen_KL_mean": -399.1279602050781, + "KL/mean": -532.711669921875, + "KL/rejected_KL_mean": -666.29541015625, + "KL/std": 251.8192901611328, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.00261482410132885, + "fcm_dpo/delta": -0.3006790280342102, + "fcm_dpo/margin": 267.1674499511719, + "fcm_dpo/q_t": 0.3441687226295471, + "grad_norm": 38.26177215576172, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": 0.18112888932228088, + "logits/rejected": 0.19067855179309845, + "logps/chosen": -459.24481201171875, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -780.241455078125, + "loss": 0.8833, + "margin_dpo/margin_mean": 267.1674499511719, + "margin_dpo/margin_std": 228.04473876953125, + "step": 465 + }, + { + "KL/chosen_KL_mean": -424.22125244140625, + "KL/mean": -533.2359619140625, + "KL/rejected_KL_mean": -642.25048828125, + "KL/std": 223.7618408203125, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0026030319277197123, + "fcm_dpo/delta": -0.1686006635427475, + "fcm_dpo/margin": 218.029296875, + "fcm_dpo/q_t": 0.3720262050628662, + "grad_norm": 39.56039810180664, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": 0.17575624585151672, + "logits/rejected": 0.18033938109874725, + "logps/chosen": -477.1422119140625, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -732.5659790039062, + "loss": 0.9836, + "margin_dpo/margin_mean": 218.02928161621094, + "margin_dpo/margin_std": 235.46994018554688, + "step": 466 + }, + { + "KL/chosen_KL_mean": -457.25921630859375, + "KL/mean": -578.416015625, + "KL/rejected_KL_mean": -699.5726928710938, + "KL/std": 249.18621826171875, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.002590528456494212, + "fcm_dpo/delta": -0.22955921292304993, + "fcm_dpo/margin": 242.3134765625, + "fcm_dpo/q_t": 0.36534398794174194, + "grad_norm": 41.40827560424805, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": 0.1942363977432251, + "logits/rejected": 0.21598802506923676, + "logps/chosen": -535.9750366210938, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -802.432861328125, + "loss": 0.9867, + "margin_dpo/margin_mean": 242.3134765625, + "margin_dpo/margin_std": 292.127685546875, + "step": 467 + }, + { + "KL/chosen_KL_mean": -377.34442138671875, + "KL/mean": -501.3637390136719, + "KL/rejected_KL_mean": -625.383056640625, + "KL/std": 235.3031768798828, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.002579476684331894, + "fcm_dpo/delta": -0.24150672554969788, + "fcm_dpo/margin": 248.03858947753906, + "fcm_dpo/q_t": 0.3575986623764038, + "grad_norm": 42.93080520629883, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": 0.11836384236812592, + "logits/rejected": 0.1588967740535736, + "logps/chosen": -447.27978515625, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -726.411865234375, + "loss": 0.9285, + "margin_dpo/margin_mean": 248.03860473632812, + "margin_dpo/margin_std": 237.18154907226562, + "step": 468 + }, + { + "KL/chosen_KL_mean": -408.3210144042969, + "KL/mean": -535.8062744140625, + "KL/rejected_KL_mean": -663.29150390625, + "KL/std": 229.92929077148438, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.002566227223724127, + "fcm_dpo/delta": -0.25619256496429443, + "fcm_dpo/margin": 254.97052001953125, + "fcm_dpo/q_t": 0.3551290035247803, + "grad_norm": 38.98451614379883, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": 0.1132114827632904, + "logits/rejected": 0.14936314523220062, + "logps/chosen": -476.4457092285156, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -768.0779418945312, + "loss": 0.921, + "margin_dpo/margin_mean": 254.97052001953125, + "margin_dpo/margin_std": 240.3369903564453, + "step": 469 + }, + { + "KL/chosen_KL_mean": -335.29901123046875, + "KL/mean": -464.4223937988281, + "KL/rejected_KL_mean": -593.5457763671875, + "KL/std": 234.0618133544922, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0025546676479279995, + "fcm_dpo/delta": -0.26142922043800354, + "fcm_dpo/margin": 258.24676513671875, + "fcm_dpo/q_t": 0.35147398710250854, + "grad_norm": 31.41411018371582, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": 0.16366428136825562, + "logits/rejected": 0.17597217857837677, + "logps/chosen": -379.0909423828125, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -676.2486572265625, + "loss": 0.9181, + "margin_dpo/margin_mean": 258.2467956542969, + "margin_dpo/margin_std": 242.90731811523438, + "step": 470 + }, + { + "KL/chosen_KL_mean": -377.30108642578125, + "KL/mean": -486.2108154296875, + "KL/rejected_KL_mean": -595.1205444335938, + "KL/std": 224.34451293945312, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0025437800213694572, + "fcm_dpo/delta": -0.15507300198078156, + "fcm_dpo/margin": 217.81947326660156, + "fcm_dpo/q_t": 0.37408214807510376, + "grad_norm": 34.01776123046875, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": 0.17552927136421204, + "logits/rejected": 0.19814151525497437, + "logps/chosen": -440.6405944824219, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -678.7310180664062, + "loss": 0.9769, + "margin_dpo/margin_mean": 217.81947326660156, + "margin_dpo/margin_std": 214.15054321289062, + "step": 471 + }, + { + "KL/chosen_KL_mean": -374.08935546875, + "KL/mean": -498.87689208984375, + "KL/rejected_KL_mean": -623.6643676757812, + "KL/std": 273.061279296875, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0025336737744510174, + "fcm_dpo/delta": -0.2338765263557434, + "fcm_dpo/margin": 249.57501220703125, + "fcm_dpo/q_t": 0.3643125295639038, + "grad_norm": 31.10460090637207, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": 0.014098504558205605, + "logits/rejected": 0.05093568563461304, + "logps/chosen": -457.7554626464844, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -740.87353515625, + "loss": 0.9525, + "margin_dpo/margin_mean": 249.57501220703125, + "margin_dpo/margin_std": 275.0196228027344, + "step": 472 + }, + { + "KL/chosen_KL_mean": -421.2077941894531, + "KL/mean": -508.77001953125, + "KL/rejected_KL_mean": -596.332275390625, + "KL/std": 223.41433715820312, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0025291151832789183, + "fcm_dpo/delta": -0.04311588779091835, + "fcm_dpo/margin": 175.12451171875, + "fcm_dpo/q_t": 0.4027405083179474, + "grad_norm": 39.13117980957031, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": 0.21204733848571777, + "logits/rejected": 0.23865394294261932, + "logps/chosen": -484.70477294921875, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -677.4788818359375, + "loss": 1.0969, + "margin_dpo/margin_mean": 175.12449645996094, + "margin_dpo/margin_std": 276.2576904296875, + "step": 473 + }, + { + "KL/chosen_KL_mean": -375.8376770019531, + "KL/mean": -474.5113220214844, + "KL/rejected_KL_mean": -573.1849365234375, + "KL/std": 231.34423828125, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.002526541007682681, + "fcm_dpo/delta": -0.09910028427839279, + "fcm_dpo/margin": 197.34725952148438, + "fcm_dpo/q_t": 0.38832181692123413, + "grad_norm": 41.12273406982422, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": 0.1530502885580063, + "logits/rejected": 0.16882580518722534, + "logps/chosen": -428.4496154785156, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -663.265380859375, + "loss": 1.0483, + "margin_dpo/margin_mean": 197.34725952148438, + "margin_dpo/margin_std": 259.2969970703125, + "step": 474 + }, + { + "KL/chosen_KL_mean": -331.48046875, + "KL/mean": -463.882568359375, + "KL/rejected_KL_mean": -596.2846069335938, + "KL/std": 227.0015869140625, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0025138319469988346, + "fcm_dpo/delta": -0.26746517419815063, + "fcm_dpo/margin": 264.80413818359375, + "fcm_dpo/q_t": 0.35150253772735596, + "grad_norm": 40.401222229003906, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": 0.30333173274993896, + "logits/rejected": 0.31088224053382874, + "logps/chosen": -373.9756774902344, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -686.3475341796875, + "loss": 0.9065, + "margin_dpo/margin_mean": 264.80413818359375, + "margin_dpo/margin_std": 239.83811950683594, + "step": 475 + }, + { + "KL/chosen_KL_mean": -361.8974304199219, + "KL/mean": -472.66522216796875, + "KL/rejected_KL_mean": -583.43310546875, + "KL/std": 216.5132293701172, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.002504445379599929, + "fcm_dpo/delta": -0.1557067632675171, + "fcm_dpo/margin": 221.53564453125, + "fcm_dpo/q_t": 0.37357521057128906, + "grad_norm": 46.281044006347656, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": 0.24882709980010986, + "logits/rejected": 0.2592979371547699, + "logps/chosen": -404.8468017578125, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -657.143310546875, + "loss": 0.97, + "margin_dpo/margin_mean": 221.53562927246094, + "margin_dpo/margin_std": 211.70953369140625, + "step": 476 + }, + { + "KL/chosen_KL_mean": -410.5400695800781, + "KL/mean": -519.1189575195312, + "KL/rejected_KL_mean": -627.6978759765625, + "KL/std": 238.85592651367188, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0024975761771202087, + "fcm_dpo/delta": -0.14313533902168274, + "fcm_dpo/margin": 217.1578369140625, + "fcm_dpo/q_t": 0.3786528408527374, + "grad_norm": 45.46604537963867, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": 0.19261857867240906, + "logits/rejected": 0.24094191193580627, + "logps/chosen": -481.31268310546875, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -703.8352661132812, + "loss": 1.011, + "margin_dpo/margin_mean": 217.1578369140625, + "margin_dpo/margin_std": 261.39361572265625, + "step": 477 + }, + { + "KL/chosen_KL_mean": -354.07171630859375, + "KL/mean": -479.7264099121094, + "KL/rejected_KL_mean": -605.381103515625, + "KL/std": 232.63584899902344, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.002488173544406891, + "fcm_dpo/delta": -0.22665968537330627, + "fcm_dpo/margin": 251.30938720703125, + "fcm_dpo/q_t": 0.3620566725730896, + "grad_norm": 33.41679000854492, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.25058478116989136, + "logits/rejected": 0.26572704315185547, + "logps/chosen": -395.5122375488281, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -690.7431030273438, + "loss": 0.9352, + "margin_dpo/margin_mean": 251.30938720703125, + "margin_dpo/margin_std": 240.16510009765625, + "step": 478 + }, + { + "KL/chosen_KL_mean": -451.81475830078125, + "KL/mean": -553.0684814453125, + "KL/rejected_KL_mean": -654.3221435546875, + "KL/std": 215.57049560546875, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.002483149990439415, + "fcm_dpo/delta": -0.10349805653095245, + "fcm_dpo/margin": 202.5074005126953, + "fcm_dpo/q_t": 0.38758182525634766, + "grad_norm": 34.67637252807617, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": 0.24256911873817444, + "logits/rejected": 0.25611960887908936, + "logps/chosen": -505.7226867675781, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -749.4384765625, + "loss": 1.044, + "margin_dpo/margin_mean": 202.5074005126953, + "margin_dpo/margin_std": 250.80267333984375, + "step": 479 + }, + { + "KL/chosen_KL_mean": -473.477783203125, + "KL/mean": -561.475830078125, + "KL/rejected_KL_mean": -649.4737548828125, + "KL/std": 236.68911743164062, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.00247942958958447, + "fcm_dpo/delta": -0.036589693278074265, + "fcm_dpo/margin": 175.99600219726562, + "fcm_dpo/q_t": 0.4028663635253906, + "grad_norm": 60.58622741699219, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": 0.3001020848751068, + "logits/rejected": 0.32833677530288696, + "logps/chosen": -532.1605224609375, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -732.40625, + "loss": 1.1032, + "margin_dpo/margin_mean": 175.99600219726562, + "margin_dpo/margin_std": 266.89990234375, + "step": 480 + }, + { + "KL/chosen_KL_mean": -417.72125244140625, + "KL/mean": -533.392822265625, + "KL/rejected_KL_mean": -649.0643310546875, + "KL/std": 253.205078125, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.002470563631504774, + "fcm_dpo/delta": -0.1725677102804184, + "fcm_dpo/margin": 231.34310913085938, + "fcm_dpo/q_t": 0.372789204120636, + "grad_norm": 48.127010345458984, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": 0.2829519808292389, + "logits/rejected": 0.29843032360076904, + "logps/chosen": -472.685546875, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -741.4848022460938, + "loss": 0.9824, + "margin_dpo/margin_mean": 231.34310913085938, + "margin_dpo/margin_std": 255.22659301757812, + "step": 481 + }, + { + "KL/chosen_KL_mean": -469.3315124511719, + "KL/mean": -552.521484375, + "KL/rejected_KL_mean": -635.71142578125, + "KL/std": 232.14419555664062, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0024685468524694443, + "fcm_dpo/delta": -0.010760626755654812, + "fcm_dpo/margin": 166.3798828125, + "fcm_dpo/q_t": 0.4063277244567871, + "grad_norm": 47.588226318359375, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": 0.20354865491390228, + "logits/rejected": 0.2115699052810669, + "logps/chosen": -536.885009765625, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -723.3009033203125, + "loss": 1.11, + "margin_dpo/margin_mean": 166.3798828125, + "margin_dpo/margin_std": 258.02203369140625, + "step": 482 + }, + { + "KL/chosen_KL_mean": -445.231201171875, + "KL/mean": -586.9732666015625, + "KL/rejected_KL_mean": -728.71533203125, + "KL/std": 262.2698669433594, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0024570925161242485, + "fcm_dpo/delta": -0.29865115880966187, + "fcm_dpo/margin": 283.484130859375, + "fcm_dpo/q_t": 0.3506658673286438, + "grad_norm": 46.95901107788086, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": 0.29704248905181885, + "logits/rejected": 0.333403080701828, + "logps/chosen": -508.5610046386719, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -824.5023193359375, + "loss": 0.9142, + "margin_dpo/margin_mean": 283.484130859375, + "margin_dpo/margin_std": 291.5053405761719, + "step": 483 + }, + { + "KL/chosen_KL_mean": -414.5421142578125, + "KL/mean": -533.795166015625, + "KL/rejected_KL_mean": -653.048095703125, + "KL/std": 245.01583862304688, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0024481462314724922, + "fcm_dpo/delta": -0.1849365085363388, + "fcm_dpo/margin": 238.50604248046875, + "fcm_dpo/q_t": 0.3697298765182495, + "grad_norm": 38.79883575439453, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": 0.34376370906829834, + "logits/rejected": 0.37063443660736084, + "logps/chosen": -473.68023681640625, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -737.4195556640625, + "loss": 0.9596, + "margin_dpo/margin_mean": 238.50604248046875, + "margin_dpo/margin_std": 233.28346252441406, + "step": 484 + }, + { + "KL/chosen_KL_mean": -405.0355529785156, + "KL/mean": -531.7224731445312, + "KL/rejected_KL_mean": -658.409423828125, + "KL/std": 254.0545654296875, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0024370343890041113, + "fcm_dpo/delta": -0.21879610419273376, + "fcm_dpo/margin": 253.3738555908203, + "fcm_dpo/q_t": 0.3640173375606537, + "grad_norm": 44.19075393676758, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": 0.2307879626750946, + "logits/rejected": 0.23904235661029816, + "logps/chosen": -463.8851318359375, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -761.7734375, + "loss": 0.9615, + "margin_dpo/margin_mean": 253.37384033203125, + "margin_dpo/margin_std": 275.82196044921875, + "step": 485 + }, + { + "KL/chosen_KL_mean": -447.776123046875, + "KL/mean": -581.5471801757812, + "KL/rejected_KL_mean": -715.3182373046875, + "KL/std": 255.3665771484375, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0024255048483610153, + "fcm_dpo/delta": -0.2504921555519104, + "fcm_dpo/margin": 267.54205322265625, + "fcm_dpo/q_t": 0.3596009612083435, + "grad_norm": 40.988197326660156, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": 0.3069908618927002, + "logits/rejected": 0.3232099413871765, + "logps/chosen": -503.0357666015625, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -807.4575805664062, + "loss": 0.9538, + "margin_dpo/margin_mean": 267.54205322265625, + "margin_dpo/margin_std": 298.17193603515625, + "step": 486 + }, + { + "KL/chosen_KL_mean": -442.2435607910156, + "KL/mean": -554.686767578125, + "KL/rejected_KL_mean": -667.1300048828125, + "KL/std": 219.71095275878906, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.002415588591247797, + "fcm_dpo/delta": -0.14409609138965607, + "fcm_dpo/margin": 224.88638305664062, + "fcm_dpo/q_t": 0.3739467263221741, + "grad_norm": 42.31475067138672, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": 0.2699805498123169, + "logits/rejected": 0.29359889030456543, + "logps/chosen": -495.306884765625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -759.548828125, + "loss": 0.9795, + "margin_dpo/margin_mean": 224.88638305664062, + "margin_dpo/margin_std": 215.0932159423828, + "step": 487 + }, + { + "KL/chosen_KL_mean": -413.37518310546875, + "KL/mean": -532.58203125, + "KL/rejected_KL_mean": -651.7890625, + "KL/std": 245.84124755859375, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0024088530335575342, + "fcm_dpo/delta": -0.17528505623340607, + "fcm_dpo/margin": 238.4138641357422, + "fcm_dpo/q_t": 0.36808228492736816, + "grad_norm": 37.23619079589844, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": 0.2703291177749634, + "logits/rejected": 0.3061204254627228, + "logps/chosen": -465.60333251953125, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -735.7955932617188, + "loss": 0.9484, + "margin_dpo/margin_mean": 238.4138641357422, + "margin_dpo/margin_std": 206.1112060546875, + "step": 488 + }, + { + "KL/chosen_KL_mean": -404.2139587402344, + "KL/mean": -503.25994873046875, + "KL/rejected_KL_mean": -602.305908203125, + "KL/std": 239.21995544433594, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0024036280810832977, + "fcm_dpo/delta": -0.07650082558393478, + "fcm_dpo/margin": 198.09194946289062, + "fcm_dpo/q_t": 0.3928455710411072, + "grad_norm": 42.459510803222656, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": 0.15185092389583588, + "logits/rejected": 0.1717294454574585, + "logps/chosen": -460.2035827636719, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -681.7039794921875, + "loss": 1.0299, + "margin_dpo/margin_mean": 198.09194946289062, + "margin_dpo/margin_std": 225.38140869140625, + "step": 489 + }, + { + "KL/chosen_KL_mean": -441.3694152832031, + "KL/mean": -557.7406616210938, + "KL/rejected_KL_mean": -674.1118774414062, + "KL/std": 294.66351318359375, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.00239734398201108, + "fcm_dpo/delta": -0.15882745385169983, + "fcm_dpo/margin": 232.74246215820312, + "fcm_dpo/q_t": 0.37906521558761597, + "grad_norm": 36.46552658081055, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": 0.2193293273448944, + "logits/rejected": 0.22708885371685028, + "logps/chosen": -493.7358093261719, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -784.52099609375, + "loss": 1.0231, + "margin_dpo/margin_mean": 232.7424774169922, + "margin_dpo/margin_std": 305.3740539550781, + "step": 490 + }, + { + "KL/chosen_KL_mean": -467.6910400390625, + "KL/mean": -556.30810546875, + "KL/rejected_KL_mean": -644.9251708984375, + "KL/std": 242.49339294433594, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.002394244307652116, + "fcm_dpo/delta": -0.024444861337542534, + "fcm_dpo/margin": 177.23416137695312, + "fcm_dpo/q_t": 0.4055347442626953, + "grad_norm": 48.10808563232422, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": 0.3123354911804199, + "logits/rejected": 0.3343454599380493, + "logps/chosen": -527.8072509765625, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -718.1979370117188, + "loss": 1.0906, + "margin_dpo/margin_mean": 177.23416137695312, + "margin_dpo/margin_std": 256.6570129394531, + "step": 491 + }, + { + "KL/chosen_KL_mean": -509.31695556640625, + "KL/mean": -580.1177978515625, + "KL/rejected_KL_mean": -650.9185791015625, + "KL/std": 233.09603881835938, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.002396009396761656, + "fcm_dpo/delta": 0.060927923768758774, + "fcm_dpo/margin": 141.6015625, + "fcm_dpo/q_t": 0.4246540367603302, + "grad_norm": 37.772396087646484, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": 0.39823949337005615, + "logits/rejected": 0.4201901853084564, + "logps/chosen": -563.3111572265625, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -723.5781860351562, + "loss": 1.1834, + "margin_dpo/margin_mean": 141.6015625, + "margin_dpo/margin_std": 285.3033142089844, + "step": 492 + }, + { + "KL/chosen_KL_mean": -505.23016357421875, + "KL/mean": -601.792236328125, + "KL/rejected_KL_mean": -698.354248046875, + "KL/std": 260.80926513671875, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0023936021607369184, + "fcm_dpo/delta": -0.06260214745998383, + "fcm_dpo/margin": 193.1241455078125, + "fcm_dpo/q_t": 0.3937148153781891, + "grad_norm": 45.98995590209961, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": 0.38034728169441223, + "logits/rejected": 0.41519761085510254, + "logps/chosen": -580.7274169921875, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -785.6773071289062, + "loss": 1.0424, + "margin_dpo/margin_mean": 193.1241455078125, + "margin_dpo/margin_std": 221.36972045898438, + "step": 493 + }, + { + "KL/chosen_KL_mean": -467.93927001953125, + "KL/mean": -603.5217895507812, + "KL/rejected_KL_mean": -739.1043090820312, + "KL/std": 268.0106201171875, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.0023851406294852495, + "fcm_dpo/delta": -0.248345747590065, + "fcm_dpo/margin": 271.1650390625, + "fcm_dpo/q_t": 0.35416555404663086, + "grad_norm": 35.87157440185547, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": 0.3657633662223816, + "logits/rejected": 0.37321826815605164, + "logps/chosen": -509.29852294921875, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -825.1956787109375, + "loss": 0.9178, + "margin_dpo/margin_mean": 271.1650390625, + "margin_dpo/margin_std": 245.6561279296875, + "step": 494 + }, + { + "KL/chosen_KL_mean": -512.09423828125, + "KL/mean": -629.7896728515625, + "KL/rejected_KL_mean": -747.4852294921875, + "KL/std": 265.4503479003906, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0023776558227837086, + "fcm_dpo/delta": -0.16055570542812347, + "fcm_dpo/margin": 235.390869140625, + "fcm_dpo/q_t": 0.3775898516178131, + "grad_norm": 36.088890075683594, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": 0.30281341075897217, + "logits/rejected": 0.31865909695625305, + "logps/chosen": -575.6293334960938, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -838.9096069335938, + "loss": 0.9979, + "margin_dpo/margin_mean": 235.390869140625, + "margin_dpo/margin_std": 274.245361328125, + "step": 495 + }, + { + "KL/chosen_KL_mean": -574.2483520507812, + "KL/mean": -663.5863647460938, + "KL/rejected_KL_mean": -752.9243774414062, + "KL/std": 236.31593322753906, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0023728730157017708, + "fcm_dpo/delta": -0.024092046543955803, + "fcm_dpo/margin": 178.676025390625, + "fcm_dpo/q_t": 0.40305548906326294, + "grad_norm": 47.71807098388672, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": 0.33693671226501465, + "logits/rejected": 0.36339497566223145, + "logps/chosen": -646.8402709960938, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -837.2537231445312, + "loss": 1.0776, + "margin_dpo/margin_mean": 178.676025390625, + "margin_dpo/margin_std": 228.1146697998047, + "step": 496 + }, + { + "KL/chosen_KL_mean": -552.5011596679688, + "KL/mean": -636.642578125, + "KL/rejected_KL_mean": -720.7838134765625, + "KL/std": 262.76446533203125, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.002371185692027211, + "fcm_dpo/delta": 0.0008407682180404663, + "fcm_dpo/margin": 168.2826690673828, + "fcm_dpo/q_t": 0.4083826541900635, + "grad_norm": 36.4189338684082, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": 0.36138373613357544, + "logits/rejected": 0.3860735297203064, + "logps/chosen": -611.0951538085938, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -797.0722045898438, + "loss": 1.0976, + "margin_dpo/margin_mean": 168.2826690673828, + "margin_dpo/margin_std": 220.60433959960938, + "step": 497 + }, + { + "KL/chosen_KL_mean": -557.47802734375, + "KL/mean": -655.2184448242188, + "KL/rejected_KL_mean": -752.9588623046875, + "KL/std": 253.092041015625, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0023709924425929785, + "fcm_dpo/delta": -0.06377653777599335, + "fcm_dpo/margin": 195.4808807373047, + "fcm_dpo/q_t": 0.39705657958984375, + "grad_norm": 41.481361389160156, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": 0.3512916564941406, + "logits/rejected": 0.3778654634952545, + "logps/chosen": -628.6836547851562, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -836.9168701171875, + "loss": 1.0791, + "margin_dpo/margin_mean": 195.48086547851562, + "margin_dpo/margin_std": 286.47747802734375, + "step": 498 + }, + { + "KL/chosen_KL_mean": -539.626953125, + "KL/mean": -674.7672119140625, + "KL/rejected_KL_mean": -809.907470703125, + "KL/std": 263.6767578125, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0023618116974830627, + "fcm_dpo/delta": -0.23986411094665527, + "fcm_dpo/margin": 270.2805480957031, + "fcm_dpo/q_t": 0.3593558669090271, + "grad_norm": 41.51131057739258, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": 0.529339075088501, + "logits/rejected": 0.5484089255332947, + "logps/chosen": -590.8821411132812, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -910.9862060546875, + "loss": 0.9533, + "margin_dpo/margin_mean": 270.2805480957031, + "margin_dpo/margin_std": 292.1595458984375, + "step": 499 + }, + { + "KL/chosen_KL_mean": -580.4674682617188, + "KL/mean": -689.1455078125, + "KL/rejected_KL_mean": -797.823486328125, + "KL/std": 253.44622802734375, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0023547238670289516, + "fcm_dpo/delta": -0.11239257454872131, + "fcm_dpo/margin": 217.35606384277344, + "fcm_dpo/q_t": 0.3860853910446167, + "grad_norm": 48.24996566772461, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": 0.4199649691581726, + "logits/rejected": 0.4187423288822174, + "logps/chosen": -637.494873046875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -891.7576904296875, + "loss": 1.0332, + "margin_dpo/margin_mean": 217.35606384277344, + "margin_dpo/margin_std": 274.9935302734375, + "step": 500 + }, + { + "KL/chosen_KL_mean": -522.4075317382812, + "KL/mean": -641.209228515625, + "KL/rejected_KL_mean": -760.010986328125, + "KL/std": 249.24462890625, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0023486316204071045, + "fcm_dpo/delta": -0.15889954566955566, + "fcm_dpo/margin": 237.60345458984375, + "fcm_dpo/q_t": 0.37646710872650146, + "grad_norm": 35.60959243774414, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": 0.4305163621902466, + "logits/rejected": 0.4711083173751831, + "logps/chosen": -576.76708984375, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -840.167724609375, + "loss": 1.0016, + "margin_dpo/margin_mean": 237.6034393310547, + "margin_dpo/margin_std": 281.41064453125, + "step": 501 + }, + { + "KL/chosen_KL_mean": -464.5247802734375, + "KL/mean": -593.3878784179688, + "KL/rejected_KL_mean": -722.2509765625, + "KL/std": 233.23281860351562, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.002339866477996111, + "fcm_dpo/delta": -0.2042257934808731, + "fcm_dpo/margin": 257.7261962890625, + "fcm_dpo/q_t": 0.361533522605896, + "grad_norm": 24.9326114654541, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": 0.2761351466178894, + "logits/rejected": 0.30950677394866943, + "logps/chosen": -532.125244140625, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -805.1997680664062, + "loss": 0.927, + "margin_dpo/margin_mean": 257.7261962890625, + "margin_dpo/margin_std": 203.23321533203125, + "step": 502 + }, + { + "KL/chosen_KL_mean": -496.0555419921875, + "KL/mean": -611.023193359375, + "KL/rejected_KL_mean": -725.9908447265625, + "KL/std": 229.543212890625, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.0023306498769670725, + "fcm_dpo/delta": -0.1367277055978775, + "fcm_dpo/margin": 229.93527221679688, + "fcm_dpo/q_t": 0.3784293532371521, + "grad_norm": 35.017520904541016, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": 0.3997420072555542, + "logits/rejected": 0.43235957622528076, + "logps/chosen": -551.1339111328125, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -808.4962768554688, + "loss": 0.9857, + "margin_dpo/margin_mean": 229.93528747558594, + "margin_dpo/margin_std": 227.48239135742188, + "step": 503 + }, + { + "KL/chosen_KL_mean": -495.9466552734375, + "KL/mean": -632.2022705078125, + "KL/rejected_KL_mean": -768.4578857421875, + "KL/std": 257.522216796875, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0023219275753945112, + "fcm_dpo/delta": -0.234230637550354, + "fcm_dpo/margin": 272.5111999511719, + "fcm_dpo/q_t": 0.3587524890899658, + "grad_norm": 31.4608154296875, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": 0.40758031606674194, + "logits/rejected": 0.4106472134590149, + "logps/chosen": -555.9124145507812, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -872.219970703125, + "loss": 0.9336, + "margin_dpo/margin_mean": 272.5111999511719, + "margin_dpo/margin_std": 253.248046875, + "step": 504 + }, + { + "KL/chosen_KL_mean": -523.21484375, + "KL/mean": -599.7017822265625, + "KL/rejected_KL_mean": -676.188720703125, + "KL/std": 269.01129150390625, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0023208351340144873, + "fcm_dpo/delta": 0.04513102024793625, + "fcm_dpo/margin": 152.97384643554688, + "fcm_dpo/q_t": 0.4203457236289978, + "grad_norm": 50.29964828491211, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": 0.3385326862335205, + "logits/rejected": 0.3602064251899719, + "logps/chosen": -599.3697509765625, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -764.7741088867188, + "loss": 1.1628, + "margin_dpo/margin_mean": 152.97384643554688, + "margin_dpo/margin_std": 282.25946044921875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -454.03570556640625, + "KL/mean": -579.9947509765625, + "KL/rejected_KL_mean": -705.953857421875, + "KL/std": 262.9169921875, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0023136246018111706, + "fcm_dpo/delta": -0.18407359719276428, + "fcm_dpo/margin": 251.91818237304688, + "fcm_dpo/q_t": 0.3688378930091858, + "grad_norm": 37.377349853515625, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": 0.3951576352119446, + "logits/rejected": 0.41258561611175537, + "logps/chosen": -502.9962158203125, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -784.368896484375, + "loss": 0.9668, + "margin_dpo/margin_mean": 251.91818237304688, + "margin_dpo/margin_std": 254.60562133789062, + "step": 506 + }, + { + "KL/chosen_KL_mean": -506.15142822265625, + "KL/mean": -610.9317626953125, + "KL/rejected_KL_mean": -715.712158203125, + "KL/std": 273.18914794921875, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0023101619444787502, + "fcm_dpo/delta": -0.08452019095420837, + "fcm_dpo/margin": 209.56072998046875, + "fcm_dpo/q_t": 0.3934593200683594, + "grad_norm": 29.474365234375, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": 0.48817938566207886, + "logits/rejected": 0.5088249444961548, + "logps/chosen": -572.9529418945312, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -811.0850830078125, + "loss": 1.0461, + "margin_dpo/margin_mean": 209.56072998046875, + "margin_dpo/margin_std": 272.718505859375, + "step": 507 + }, + { + "KL/chosen_KL_mean": -491.5720520019531, + "KL/mean": -588.209716796875, + "KL/rejected_KL_mean": -684.847412109375, + "KL/std": 274.30865478515625, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0023078611120581627, + "fcm_dpo/delta": -0.04626144468784332, + "fcm_dpo/margin": 193.275390625, + "fcm_dpo/q_t": 0.40172165632247925, + "grad_norm": 39.37025451660156, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": 0.3990686535835266, + "logits/rejected": 0.4105161726474762, + "logps/chosen": -562.87548828125, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -780.4749755859375, + "loss": 1.0841, + "margin_dpo/margin_mean": 193.275390625, + "margin_dpo/margin_std": 279.6917724609375, + "step": 508 + }, + { + "KL/chosen_KL_mean": -437.29742431640625, + "KL/mean": -531.2692260742188, + "KL/rejected_KL_mean": -625.2409057617188, + "KL/std": 231.4613037109375, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.002306253183633089, + "fcm_dpo/delta": -0.0335998609662056, + "fcm_dpo/margin": 187.9434814453125, + "fcm_dpo/q_t": 0.40263718366622925, + "grad_norm": 46.70771789550781, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": 0.24978551268577576, + "logits/rejected": 0.25812727212905884, + "logps/chosen": -501.11639404296875, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -708.4973754882812, + "loss": 1.0732, + "margin_dpo/margin_mean": 187.9434814453125, + "margin_dpo/margin_std": 244.68536376953125, + "step": 509 + }, + { + "KL/chosen_KL_mean": -492.68585205078125, + "KL/mean": -617.6367797851562, + "KL/rejected_KL_mean": -742.5877685546875, + "KL/std": 238.43148803710938, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.002299422863870859, + "fcm_dpo/delta": -0.1756180226802826, + "fcm_dpo/margin": 249.90187072753906, + "fcm_dpo/q_t": 0.36856213212013245, + "grad_norm": 34.4243278503418, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": 0.36153674125671387, + "logits/rejected": 0.36188966035842896, + "logps/chosen": -544.5643310546875, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -845.3529052734375, + "loss": 0.9561, + "margin_dpo/margin_mean": 249.90185546875, + "margin_dpo/margin_std": 229.6932373046875, + "step": 510 + }, + { + "KL/chosen_KL_mean": -475.92431640625, + "KL/mean": -596.5877685546875, + "KL/rejected_KL_mean": -717.251220703125, + "KL/std": 270.0506896972656, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.002292448654770851, + "fcm_dpo/delta": -0.15405380725860596, + "fcm_dpo/margin": 241.32687377929688, + "fcm_dpo/q_t": 0.3775292932987213, + "grad_norm": 40.84492492675781, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": 0.45296788215637207, + "logits/rejected": 0.46452999114990234, + "logps/chosen": -536.1624755859375, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -810.1079711914062, + "loss": 1.014, + "margin_dpo/margin_mean": 241.32687377929688, + "margin_dpo/margin_std": 299.561279296875, + "step": 511 + }, + { + "KL/chosen_KL_mean": -443.09716796875, + "KL/mean": -572.7542114257812, + "KL/rejected_KL_mean": -702.4112548828125, + "KL/std": 245.21214294433594, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.002283469308167696, + "fcm_dpo/delta": -0.19327253103256226, + "fcm_dpo/margin": 259.3140563964844, + "fcm_dpo/q_t": 0.36533647775650024, + "grad_norm": 26.426612854003906, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.5010051131248474, + "logits/rejected": 0.5280622243881226, + "logps/chosen": -498.0026550292969, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -784.287109375, + "loss": 0.9464, + "margin_dpo/margin_mean": 259.31402587890625, + "margin_dpo/margin_std": 238.35928344726562, + "step": 512 + }, + { + "KL/chosen_KL_mean": -487.9002685546875, + "KL/mean": -582.6265258789062, + "KL/rejected_KL_mean": -677.352783203125, + "KL/std": 233.59628295898438, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0022783055901527405, + "fcm_dpo/delta": -0.03191276639699936, + "fcm_dpo/margin": 189.45245361328125, + "fcm_dpo/q_t": 0.40161728858947754, + "grad_norm": 60.03765106201172, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": 0.5188250541687012, + "logits/rejected": 0.5511025190353394, + "logps/chosen": -552.8167114257812, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -753.4151611328125, + "loss": 1.089, + "margin_dpo/margin_mean": 189.45245361328125, + "margin_dpo/margin_std": 262.35888671875, + "step": 513 + }, + { + "KL/chosen_KL_mean": -487.2889709472656, + "KL/mean": -579.3116455078125, + "KL/rejected_KL_mean": -671.334228515625, + "KL/std": 232.11480712890625, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.0022791558876633644, + "fcm_dpo/delta": -0.019549354910850525, + "fcm_dpo/margin": 184.04525756835938, + "fcm_dpo/q_t": 0.4060305953025818, + "grad_norm": 37.08213424682617, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": 0.34559887647628784, + "logits/rejected": 0.363470196723938, + "logps/chosen": -561.5185546875, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -750.27978515625, + "loss": 1.1026, + "margin_dpo/margin_mean": 184.04525756835938, + "margin_dpo/margin_std": 279.2259521484375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -419.930908203125, + "KL/mean": -510.17315673828125, + "KL/rejected_KL_mean": -600.415283203125, + "KL/std": 231.27606201171875, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.002277845051139593, + "fcm_dpo/delta": -0.011177672073245049, + "fcm_dpo/margin": 180.48440551757812, + "fcm_dpo/q_t": 0.40453702211380005, + "grad_norm": 39.98896408081055, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": 0.31468841433525085, + "logits/rejected": 0.31995755434036255, + "logps/chosen": -470.3324890136719, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -687.5130615234375, + "loss": 1.0824, + "margin_dpo/margin_mean": 180.48440551757812, + "margin_dpo/margin_std": 230.99517822265625, + "step": 515 + }, + { + "KL/chosen_KL_mean": -447.2619934082031, + "KL/mean": -574.1876220703125, + "KL/rejected_KL_mean": -701.11328125, + "KL/std": 251.75253295898438, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.002273065969347954, + "fcm_dpo/delta": -0.17800989747047424, + "fcm_dpo/margin": 253.851318359375, + "fcm_dpo/q_t": 0.3688949942588806, + "grad_norm": 31.907310485839844, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": 0.4788651466369629, + "logits/rejected": 0.4791467487812042, + "logps/chosen": -511.91156005859375, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -812.835693359375, + "loss": 0.9601, + "margin_dpo/margin_mean": 253.851318359375, + "margin_dpo/margin_std": 242.0334014892578, + "step": 516 + }, + { + "KL/chosen_KL_mean": -479.153564453125, + "KL/mean": -605.927490234375, + "KL/rejected_KL_mean": -732.701416015625, + "KL/std": 266.4066162109375, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.002263226779177785, + "fcm_dpo/delta": -0.17496204376220703, + "fcm_dpo/margin": 253.54788208007812, + "fcm_dpo/q_t": 0.373352587223053, + "grad_norm": 31.713314056396484, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": 0.36331599950790405, + "logits/rejected": 0.3876442313194275, + "logps/chosen": -540.067138671875, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -821.7844848632812, + "loss": 0.9831, + "margin_dpo/margin_mean": 253.54788208007812, + "margin_dpo/margin_std": 282.049560546875, + "step": 517 + }, + { + "KL/chosen_KL_mean": -451.661376953125, + "KL/mean": -560.8165283203125, + "KL/rejected_KL_mean": -669.9717407226562, + "KL/std": 225.25225830078125, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.002258509863168001, + "fcm_dpo/delta": -0.09355603158473969, + "fcm_dpo/margin": 218.31031799316406, + "fcm_dpo/q_t": 0.3875216841697693, + "grad_norm": 36.60832977294922, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": 0.3798970878124237, + "logits/rejected": 0.3902730345726013, + "logps/chosen": -509.1172790527344, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -755.284423828125, + "loss": 1.034, + "margin_dpo/margin_mean": 218.31031799316406, + "margin_dpo/margin_std": 262.9700622558594, + "step": 518 + }, + { + "KL/chosen_KL_mean": -445.78558349609375, + "KL/mean": -537.140625, + "KL/rejected_KL_mean": -628.4957275390625, + "KL/std": 246.82656860351562, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0022582761012017727, + "fcm_dpo/delta": -0.012675488367676735, + "fcm_dpo/margin": 182.71017456054688, + "fcm_dpo/q_t": 0.4067375659942627, + "grad_norm": 34.598121643066406, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": 0.3192419409751892, + "logits/rejected": 0.33089736104011536, + "logps/chosen": -519.848876953125, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -732.93994140625, + "loss": 1.0973, + "margin_dpo/margin_mean": 182.71018981933594, + "margin_dpo/margin_std": 260.5166931152344, + "step": 519 + }, + { + "KL/chosen_KL_mean": -446.1053771972656, + "KL/mean": -551.1536865234375, + "KL/rejected_KL_mean": -656.2021484375, + "KL/std": 239.96932983398438, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.002253470243886113, + "fcm_dpo/delta": -0.07393531501293182, + "fcm_dpo/margin": 210.09678649902344, + "fcm_dpo/q_t": 0.3927437961101532, + "grad_norm": 27.411405563354492, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": 0.3782256841659546, + "logits/rejected": 0.3861110210418701, + "logps/chosen": -516.4052124023438, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -756.1834716796875, + "loss": 1.0361, + "margin_dpo/margin_mean": 210.0968017578125, + "margin_dpo/margin_std": 230.12594604492188, + "step": 520 + }, + { + "KL/chosen_KL_mean": -432.99835205078125, + "KL/mean": -536.646484375, + "KL/rejected_KL_mean": -640.2945556640625, + "KL/std": 234.36184692382812, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0022536139003932476, + "fcm_dpo/delta": -0.06754961609840393, + "fcm_dpo/margin": 207.29624938964844, + "fcm_dpo/q_t": 0.39347681403160095, + "grad_norm": 46.18112564086914, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": 0.4619916081428528, + "logits/rejected": 0.4870032072067261, + "logps/chosen": -491.1412658691406, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -723.5751953125, + "loss": 1.0469, + "margin_dpo/margin_mean": 207.29624938964844, + "margin_dpo/margin_std": 242.404296875, + "step": 521 + }, + { + "KL/chosen_KL_mean": -457.6671142578125, + "KL/mean": -554.2105712890625, + "KL/rejected_KL_mean": -650.7540893554688, + "KL/std": 225.99530029296875, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0022500581108033657, + "fcm_dpo/delta": -0.03460593521595001, + "fcm_dpo/margin": 193.0869598388672, + "fcm_dpo/q_t": 0.39956000447273254, + "grad_norm": 33.22862243652344, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": 0.47090816497802734, + "logits/rejected": 0.48173242807388306, + "logps/chosen": -504.21478271484375, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -716.7679443359375, + "loss": 1.0684, + "margin_dpo/margin_mean": 193.0869598388672, + "margin_dpo/margin_std": 242.58660888671875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -456.9018859863281, + "KL/mean": -563.4923095703125, + "KL/rejected_KL_mean": -670.082763671875, + "KL/std": 254.59683227539062, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0022468469105660915, + "fcm_dpo/delta": -0.07936576008796692, + "fcm_dpo/margin": 213.180908203125, + "fcm_dpo/q_t": 0.38905125856399536, + "grad_norm": 43.7906494140625, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": 0.41408759355545044, + "logits/rejected": 0.45472216606140137, + "logps/chosen": -518.6715087890625, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -753.84423828125, + "loss": 1.0428, + "margin_dpo/margin_mean": 213.180908203125, + "margin_dpo/margin_std": 260.38726806640625, + "step": 523 + }, + { + "KL/chosen_KL_mean": -469.2633056640625, + "KL/mean": -578.8290405273438, + "KL/rejected_KL_mean": -688.394775390625, + "KL/std": 247.5463409423828, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0022417926229536533, + "fcm_dpo/delta": -0.09179022163152695, + "fcm_dpo/margin": 219.13150024414062, + "fcm_dpo/q_t": 0.38579684495925903, + "grad_norm": 33.425987243652344, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": 0.41422897577285767, + "logits/rejected": 0.46197566390037537, + "logps/chosen": -547.3353271484375, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -769.69677734375, + "loss": 1.0179, + "margin_dpo/margin_mean": 219.13150024414062, + "margin_dpo/margin_std": 227.67623901367188, + "step": 524 + }, + { + "KL/chosen_KL_mean": -450.8065185546875, + "KL/mean": -592.179931640625, + "KL/rejected_KL_mean": -733.553466796875, + "KL/std": 275.8620910644531, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0022325399331748486, + "fcm_dpo/delta": -0.2329603135585785, + "fcm_dpo/margin": 282.7469482421875, + "fcm_dpo/q_t": 0.3584517538547516, + "grad_norm": 35.123470306396484, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": 0.45596858859062195, + "logits/rejected": 0.4658408761024475, + "logps/chosen": -501.6343688964844, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -833.6063842773438, + "loss": 0.9277, + "margin_dpo/margin_mean": 282.7469482421875, + "margin_dpo/margin_std": 257.96307373046875, + "step": 525 + }, + { + "KL/chosen_KL_mean": -462.151611328125, + "KL/mean": -593.2882080078125, + "KL/rejected_KL_mean": -724.4248046875, + "KL/std": 255.14073181152344, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.00222639087587595, + "fcm_dpo/delta": -0.1849607676267624, + "fcm_dpo/margin": 262.273193359375, + "fcm_dpo/q_t": 0.3676462769508362, + "grad_norm": 33.104522705078125, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": 0.3517810106277466, + "logits/rejected": 0.38328516483306885, + "logps/chosen": -525.31884765625, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -810.734130859375, + "loss": 0.9577, + "margin_dpo/margin_mean": 262.273193359375, + "margin_dpo/margin_std": 250.69107055664062, + "step": 526 + }, + { + "KL/chosen_KL_mean": -486.051513671875, + "KL/mean": -590.2175903320312, + "KL/rejected_KL_mean": -694.3836669921875, + "KL/std": 253.55264282226562, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0022201836109161377, + "fcm_dpo/delta": -0.06288629025220871, + "fcm_dpo/margin": 208.33218383789062, + "fcm_dpo/q_t": 0.39355844259262085, + "grad_norm": 31.775182723999023, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": 0.6181141138076782, + "logits/rejected": 0.6457891464233398, + "logps/chosen": -557.18017578125, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -772.7262573242188, + "loss": 1.0355, + "margin_dpo/margin_mean": 208.33216857910156, + "margin_dpo/margin_std": 224.7138671875, + "step": 527 + }, + { + "KL/chosen_KL_mean": -501.57098388671875, + "KL/mean": -610.4314575195312, + "KL/rejected_KL_mean": -719.2919311523438, + "KL/std": 260.2093505859375, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0022174103651195765, + "fcm_dpo/delta": -0.08318869769573212, + "fcm_dpo/margin": 217.72097778320312, + "fcm_dpo/q_t": 0.39245256781578064, + "grad_norm": 39.62614440917969, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": 0.49699753522872925, + "logits/rejected": 0.5205287933349609, + "logps/chosen": -569.660400390625, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -813.2020263671875, + "loss": 1.0504, + "margin_dpo/margin_mean": 217.72097778320312, + "margin_dpo/margin_std": 284.96826171875, + "step": 528 + }, + { + "KL/chosen_KL_mean": -575.0556030273438, + "KL/mean": -661.8389282226562, + "KL/rejected_KL_mean": -748.6222534179688, + "KL/std": 233.74227905273438, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.002217243891209364, + "fcm_dpo/delta": 0.015218764543533325, + "fcm_dpo/margin": 173.566650390625, + "fcm_dpo/q_t": 0.4118032157421112, + "grad_norm": 43.196434020996094, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": 0.6185893416404724, + "logits/rejected": 0.6164660453796387, + "logps/chosen": -630.630615234375, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -837.8313598632812, + "loss": 1.1233, + "margin_dpo/margin_mean": 173.566650390625, + "margin_dpo/margin_std": 268.6534118652344, + "step": 529 + }, + { + "KL/chosen_KL_mean": -490.29119873046875, + "KL/mean": -623.7972412109375, + "KL/rejected_KL_mean": -757.30322265625, + "KL/std": 264.4576721191406, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.002210577018558979, + "fcm_dpo/delta": -0.19142815470695496, + "fcm_dpo/margin": 267.01202392578125, + "fcm_dpo/q_t": 0.36837008595466614, + "grad_norm": 40.6228141784668, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": 0.4926597476005554, + "logits/rejected": 0.5052857398986816, + "logps/chosen": -537.892578125, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -844.5877685546875, + "loss": 0.9804, + "margin_dpo/margin_mean": 267.01202392578125, + "margin_dpo/margin_std": 299.77398681640625, + "step": 530 + }, + { + "KL/chosen_KL_mean": -521.332763671875, + "KL/mean": -640.3416748046875, + "KL/rejected_KL_mean": -759.3505249023438, + "KL/std": 320.8760986328125, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.002204917836934328, + "fcm_dpo/delta": -0.12545998394489288, + "fcm_dpo/margin": 238.01768493652344, + "fcm_dpo/q_t": 0.3822554647922516, + "grad_norm": 44.48646926879883, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": 0.540439248085022, + "logits/rejected": 0.56269770860672, + "logps/chosen": -576.578857421875, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -829.95654296875, + "loss": 1.0042, + "margin_dpo/margin_mean": 238.0177001953125, + "margin_dpo/margin_std": 263.8735046386719, + "step": 531 + }, + { + "KL/chosen_KL_mean": -559.7723388671875, + "KL/mean": -680.4659423828125, + "KL/rejected_KL_mean": -801.1594848632812, + "KL/std": 285.09503173828125, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0021972041577100754, + "fcm_dpo/delta": -0.13132211565971375, + "fcm_dpo/margin": 241.38719177246094, + "fcm_dpo/q_t": 0.383047878742218, + "grad_norm": 50.45771408081055, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": 0.5104477405548096, + "logits/rejected": 0.5358579158782959, + "logps/chosen": -630.058349609375, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -887.7508544921875, + "loss": 1.0479, + "margin_dpo/margin_mean": 241.38717651367188, + "margin_dpo/margin_std": 329.48046875, + "step": 532 + }, + { + "KL/chosen_KL_mean": -535.8917236328125, + "KL/mean": -638.7140502929688, + "KL/rejected_KL_mean": -741.536376953125, + "KL/std": 264.57684326171875, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0021974900737404823, + "fcm_dpo/delta": -0.05220738425850868, + "fcm_dpo/margin": 205.64462280273438, + "fcm_dpo/q_t": 0.4004591703414917, + "grad_norm": 42.118141174316406, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": 0.6520870923995972, + "logits/rejected": 0.6815459728240967, + "logps/chosen": -600.746826171875, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -818.1244506835938, + "loss": 1.0827, + "margin_dpo/margin_mean": 205.64462280273438, + "margin_dpo/margin_std": 292.395263671875, + "step": 533 + }, + { + "KL/chosen_KL_mean": -569.033935546875, + "KL/mean": -670.2034912109375, + "KL/rejected_KL_mean": -771.3731079101562, + "KL/std": 276.14898681640625, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0021936013363301754, + "fcm_dpo/delta": -0.04405461996793747, + "fcm_dpo/margin": 202.33917236328125, + "fcm_dpo/q_t": 0.39828336238861084, + "grad_norm": 51.987709045410156, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": 0.5690357685089111, + "logits/rejected": 0.584942102432251, + "logps/chosen": -629.1533203125, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -849.9165649414062, + "loss": 1.0681, + "margin_dpo/margin_mean": 202.33917236328125, + "margin_dpo/margin_std": 263.13385009765625, + "step": 534 + }, + { + "KL/chosen_KL_mean": -504.7890625, + "KL/mean": -636.3511962890625, + "KL/rejected_KL_mean": -767.9132690429688, + "KL/std": 273.5988464355469, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0021889209747314453, + "fcm_dpo/delta": -0.17694474756717682, + "fcm_dpo/margin": 263.1241455078125, + "fcm_dpo/q_t": 0.37040793895721436, + "grad_norm": 35.55424880981445, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": 0.5183975696563721, + "logits/rejected": 0.5338696241378784, + "logps/chosen": -559.1193237304688, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -864.2208862304688, + "loss": 0.967, + "margin_dpo/margin_mean": 263.1241455078125, + "margin_dpo/margin_std": 258.5193786621094, + "step": 535 + }, + { + "KL/chosen_KL_mean": -460.5835266113281, + "KL/mean": -616.671630859375, + "KL/rejected_KL_mean": -772.7596435546875, + "KL/std": 316.2499694824219, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0021764400880783796, + "fcm_dpo/delta": -0.28134265542030334, + "fcm_dpo/margin": 312.1761474609375, + "fcm_dpo/q_t": 0.3541851043701172, + "grad_norm": 31.101680755615234, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": 0.5393592715263367, + "logits/rejected": 0.5677004456520081, + "logps/chosen": -507.6640625, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -861.8575439453125, + "loss": 0.9168, + "margin_dpo/margin_mean": 312.1761474609375, + "margin_dpo/margin_std": 325.3968505859375, + "step": 536 + }, + { + "KL/chosen_KL_mean": -509.994873046875, + "KL/mean": -632.9329833984375, + "KL/rejected_KL_mean": -755.8710327148438, + "KL/std": 261.60833740234375, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.002169529441744089, + "fcm_dpo/delta": -0.13413016498088837, + "fcm_dpo/margin": 245.87612915039062, + "fcm_dpo/q_t": 0.38036900758743286, + "grad_norm": 40.3139533996582, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": 0.6547621488571167, + "logits/rejected": 0.6678478717803955, + "logps/chosen": -567.7423706054688, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -826.3094482421875, + "loss": 1.0138, + "margin_dpo/margin_mean": 245.87611389160156, + "margin_dpo/margin_std": 291.785888671875, + "step": 537 + }, + { + "KL/chosen_KL_mean": -560.429931640625, + "KL/mean": -680.0866088867188, + "KL/rejected_KL_mean": -799.7432861328125, + "KL/std": 262.1174621582031, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0021632518619298935, + "fcm_dpo/delta": -0.11834007501602173, + "fcm_dpo/margin": 239.31338500976562, + "fcm_dpo/q_t": 0.38276779651641846, + "grad_norm": 66.82955169677734, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": 0.4910144805908203, + "logits/rejected": 0.5067077279090881, + "logps/chosen": -626.8458251953125, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -883.971435546875, + "loss": 1.0096, + "margin_dpo/margin_mean": 239.3134002685547, + "margin_dpo/margin_std": 267.09130859375, + "step": 538 + }, + { + "KL/chosen_KL_mean": -536.5831298828125, + "KL/mean": -650.9773559570312, + "KL/rejected_KL_mean": -765.3715209960938, + "KL/std": 257.0419616699219, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0021590932738035917, + "fcm_dpo/delta": -0.09445612132549286, + "fcm_dpo/margin": 228.7884063720703, + "fcm_dpo/q_t": 0.3877899646759033, + "grad_norm": 42.96073913574219, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": 0.620707631111145, + "logits/rejected": 0.6563342809677124, + "logps/chosen": -595.075927734375, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -857.2254638671875, + "loss": 1.0289, + "margin_dpo/margin_mean": 228.78842163085938, + "margin_dpo/margin_std": 266.78173828125, + "step": 539 + }, + { + "KL/chosen_KL_mean": -522.8631591796875, + "KL/mean": -681.39990234375, + "KL/rejected_KL_mean": -839.9365234375, + "KL/std": 321.80828857421875, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0021500340662896633, + "fcm_dpo/delta": -0.2835712432861328, + "fcm_dpo/margin": 317.07330322265625, + "fcm_dpo/q_t": 0.3522881269454956, + "grad_norm": 38.67402648925781, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": 0.4684346318244934, + "logits/rejected": 0.4886873960494995, + "logps/chosen": -586.345703125, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -956.3665161132812, + "loss": 0.9308, + "margin_dpo/margin_mean": 317.07330322265625, + "margin_dpo/margin_std": 328.336181640625, + "step": 540 + }, + { + "KL/chosen_KL_mean": -576.0125732421875, + "KL/mean": -676.1700439453125, + "KL/rejected_KL_mean": -776.3275146484375, + "KL/std": 283.4687194824219, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0021435320377349854, + "fcm_dpo/delta": -0.029647454619407654, + "fcm_dpo/margin": 200.31488037109375, + "fcm_dpo/q_t": 0.40524640679359436, + "grad_norm": 38.67329788208008, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": 0.6274768114089966, + "logits/rejected": 0.6367487907409668, + "logps/chosen": -654.29296875, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -879.6011962890625, + "loss": 1.0952, + "margin_dpo/margin_mean": 200.31488037109375, + "margin_dpo/margin_std": 292.97308349609375, + "step": 541 + }, + { + "KL/chosen_KL_mean": -561.5970458984375, + "KL/mean": -700.8912353515625, + "KL/rejected_KL_mean": -840.1854248046875, + "KL/std": 297.84552001953125, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.002137484261766076, + "fcm_dpo/delta": -0.19679415225982666, + "fcm_dpo/margin": 278.58831787109375, + "fcm_dpo/q_t": 0.36839449405670166, + "grad_norm": 32.767940521240234, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": 0.6840941905975342, + "logits/rejected": 0.7097320556640625, + "logps/chosen": -619.08203125, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -936.6605224609375, + "loss": 0.983, + "margin_dpo/margin_mean": 278.58831787109375, + "margin_dpo/margin_std": 319.32763671875, + "step": 542 + }, + { + "KL/chosen_KL_mean": -578.3162231445312, + "KL/mean": -680.9368896484375, + "KL/rejected_KL_mean": -783.5576171875, + "KL/std": 337.57745361328125, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0021345634013414383, + "fcm_dpo/delta": -0.038297537714242935, + "fcm_dpo/margin": 205.24139404296875, + "fcm_dpo/q_t": 0.4081553816795349, + "grad_norm": 44.27385330200195, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": 0.5311863422393799, + "logits/rejected": 0.5430483818054199, + "logps/chosen": -638.9337158203125, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -889.1566162109375, + "loss": 1.1122, + "margin_dpo/margin_mean": 205.24139404296875, + "margin_dpo/margin_std": 345.15423583984375, + "step": 543 + }, + { + "KL/chosen_KL_mean": -587.4173583984375, + "KL/mean": -730.015380859375, + "KL/rejected_KL_mean": -872.6133422851562, + "KL/std": 304.6583251953125, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0021284238900989294, + "fcm_dpo/delta": -0.20825795829296112, + "fcm_dpo/margin": 285.19598388671875, + "fcm_dpo/q_t": 0.3648024797439575, + "grad_norm": 36.95684051513672, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": 0.6051541566848755, + "logits/rejected": 0.6286282539367676, + "logps/chosen": -647.0596923828125, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -973.5680541992188, + "loss": 0.9502, + "margin_dpo/margin_mean": 285.19598388671875, + "margin_dpo/margin_std": 288.7699279785156, + "step": 544 + }, + { + "KL/chosen_KL_mean": -582.6710205078125, + "KL/mean": -723.1217041015625, + "KL/rejected_KL_mean": -863.572265625, + "KL/std": 290.3184814453125, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.002119791693985462, + "fcm_dpo/delta": -0.1966116726398468, + "fcm_dpo/margin": 280.9012451171875, + "fcm_dpo/q_t": 0.36670106649398804, + "grad_norm": 31.305051803588867, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": 0.525539219379425, + "logits/rejected": 0.5552965402603149, + "logps/chosen": -650.3196411132812, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -959.4802856445312, + "loss": 0.96, + "margin_dpo/margin_mean": 280.90118408203125, + "margin_dpo/margin_std": 286.6955871582031, + "step": 545 + }, + { + "KL/chosen_KL_mean": -539.624267578125, + "KL/mean": -653.2982177734375, + "KL/rejected_KL_mean": -766.97216796875, + "KL/std": 264.64691162109375, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.002114463597536087, + "fcm_dpo/delta": -0.0811421126127243, + "fcm_dpo/margin": 227.347900390625, + "fcm_dpo/q_t": 0.38750457763671875, + "grad_norm": 32.54119110107422, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.5742855668067932, + "logits/rejected": 0.5759142637252808, + "logps/chosen": -590.3684692382812, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -848.83837890625, + "loss": 1.0285, + "margin_dpo/margin_mean": 227.34791564941406, + "margin_dpo/margin_std": 249.912353515625, + "step": 546 + }, + { + "KL/chosen_KL_mean": -593.1096801757812, + "KL/mean": -709.7387084960938, + "KL/rejected_KL_mean": -826.3677978515625, + "KL/std": 302.6866760253906, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0021118447184562683, + "fcm_dpo/delta": -0.09305369108915329, + "fcm_dpo/margin": 233.25808715820312, + "fcm_dpo/q_t": 0.3916705548763275, + "grad_norm": 49.80030822753906, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": 0.46291160583496094, + "logits/rejected": 0.49829137325286865, + "logps/chosen": -666.79736328125, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -917.129150390625, + "loss": 1.058, + "margin_dpo/margin_mean": 233.25807189941406, + "margin_dpo/margin_std": 320.32000732421875, + "step": 547 + }, + { + "KL/chosen_KL_mean": -596.2838134765625, + "KL/mean": -720.1834716796875, + "KL/rejected_KL_mean": -844.0831298828125, + "KL/std": 321.8070068359375, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0021064942702651024, + "fcm_dpo/delta": -0.12261981517076492, + "fcm_dpo/margin": 247.7993621826172, + "fcm_dpo/q_t": 0.3874475955963135, + "grad_norm": 36.29145812988281, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": 0.5478363037109375, + "logits/rejected": 0.5779159665107727, + "logps/chosen": -661.5301513671875, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -938.201171875, + "loss": 1.0334, + "margin_dpo/margin_mean": 247.79934692382812, + "margin_dpo/margin_std": 331.444580078125, + "step": 548 + }, + { + "KL/chosen_KL_mean": -596.2221069335938, + "KL/mean": -705.0183715820312, + "KL/rejected_KL_mean": -813.8146362304688, + "KL/std": 248.502197265625, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0021038930863142014, + "fcm_dpo/delta": -0.05805947259068489, + "fcm_dpo/margin": 217.59249877929688, + "fcm_dpo/q_t": 0.39680415391921997, + "grad_norm": 40.39864730834961, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": 0.7528241872787476, + "logits/rejected": 0.7703467607498169, + "logps/chosen": -645.4344482421875, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -887.7249755859375, + "loss": 1.0629, + "margin_dpo/margin_mean": 217.59249877929688, + "margin_dpo/margin_std": 284.3935241699219, + "step": 549 + }, + { + "KL/chosen_KL_mean": -581.383544921875, + "KL/mean": -688.9580688476562, + "KL/rejected_KL_mean": -796.5325927734375, + "KL/std": 295.2151184082031, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.0021004220470786095, + "fcm_dpo/delta": -0.05216540768742561, + "fcm_dpo/margin": 215.14906311035156, + "fcm_dpo/q_t": 0.3958188593387604, + "grad_norm": 34.3386344909668, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": 0.5369948148727417, + "logits/rejected": 0.5540516376495361, + "logps/chosen": -638.1905517578125, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -891.658447265625, + "loss": 1.0554, + "margin_dpo/margin_mean": 215.14907836914062, + "margin_dpo/margin_std": 262.144287109375, + "step": 550 + }, + { + "KL/chosen_KL_mean": -577.2806396484375, + "KL/mean": -734.6836547851562, + "KL/rejected_KL_mean": -892.0867309570312, + "KL/std": 316.8522033691406, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0020922794938087463, + "fcm_dpo/delta": -0.2603708505630493, + "fcm_dpo/margin": 314.8060607910156, + "fcm_dpo/q_t": 0.35170382261276245, + "grad_norm": 38.978553771972656, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": 0.5319101214408875, + "logits/rejected": 0.5578449964523315, + "logps/chosen": -636.386962890625, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1003.759521484375, + "loss": 0.911, + "margin_dpo/margin_mean": 314.8060607910156, + "margin_dpo/margin_std": 280.67181396484375, + "step": 551 + }, + { + "KL/chosen_KL_mean": -551.0818481445312, + "KL/mean": -738.8308715820312, + "KL/rejected_KL_mean": -926.5798950195312, + "KL/std": 341.516357421875, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0020752931013703346, + "fcm_dpo/delta": -0.38289839029312134, + "fcm_dpo/margin": 375.4980773925781, + "fcm_dpo/q_t": 0.3357250690460205, + "grad_norm": 30.991436004638672, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": 0.6441129446029663, + "logits/rejected": 0.6995079517364502, + "logps/chosen": -613.4364624023438, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1031.14208984375, + "loss": 0.8757, + "margin_dpo/margin_mean": 375.498046875, + "margin_dpo/margin_std": 360.6898193359375, + "step": 552 + }, + { + "KL/chosen_KL_mean": -582.6217041015625, + "KL/mean": -693.4592895507812, + "KL/rejected_KL_mean": -804.296875, + "KL/std": 293.1231994628906, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.002071223221719265, + "fcm_dpo/delta": -0.05943654105067253, + "fcm_dpo/margin": 221.67523193359375, + "fcm_dpo/q_t": 0.3986949920654297, + "grad_norm": 42.23432540893555, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": 0.5881531238555908, + "logits/rejected": 0.5936106443405151, + "logps/chosen": -650.8804931640625, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -902.39404296875, + "loss": 1.0743, + "margin_dpo/margin_mean": 221.67523193359375, + "margin_dpo/margin_std": 312.017822265625, + "step": 553 + }, + { + "KL/chosen_KL_mean": -589.2076416015625, + "KL/mean": -746.763916015625, + "KL/rejected_KL_mean": -904.320068359375, + "KL/std": 339.2746887207031, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0020645284093916416, + "fcm_dpo/delta": -0.2521201968193054, + "fcm_dpo/margin": 315.1124267578125, + "fcm_dpo/q_t": 0.3604927659034729, + "grad_norm": 39.45507049560547, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": 0.5246318578720093, + "logits/rejected": 0.574246346950531, + "logps/chosen": -657.1553955078125, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -994.102783203125, + "loss": 0.9646, + "margin_dpo/margin_mean": 315.1124267578125, + "margin_dpo/margin_std": 369.6709899902344, + "step": 554 + }, + { + "KL/chosen_KL_mean": -586.3340454101562, + "KL/mean": -746.2847900390625, + "KL/rejected_KL_mean": -906.235595703125, + "KL/std": 319.7850341796875, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.002052393974736333, + "fcm_dpo/delta": -0.25830507278442383, + "fcm_dpo/margin": 319.901611328125, + "fcm_dpo/q_t": 0.36057138442993164, + "grad_norm": 50.90303039550781, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": 0.5350759625434875, + "logits/rejected": 0.5654920339584351, + "logps/chosen": -639.66455078125, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1014.7149658203125, + "loss": 0.9549, + "margin_dpo/margin_mean": 319.901611328125, + "margin_dpo/margin_std": 369.529052734375, + "step": 555 + }, + { + "KL/chosen_KL_mean": -598.8048095703125, + "KL/mean": -712.546875, + "KL/rejected_KL_mean": -826.2889404296875, + "KL/std": 290.81805419921875, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0020480165258049965, + "fcm_dpo/delta": -0.06619983911514282, + "fcm_dpo/margin": 227.48411560058594, + "fcm_dpo/q_t": 0.39653217792510986, + "grad_norm": 31.08798599243164, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": 0.6976491808891296, + "logits/rejected": 0.7135498523712158, + "logps/chosen": -657.4492797851562, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -927.6292724609375, + "loss": 1.0544, + "margin_dpo/margin_mean": 227.484130859375, + "margin_dpo/margin_std": 293.13031005859375, + "step": 556 + }, + { + "KL/chosen_KL_mean": -624.8375244140625, + "KL/mean": -763.0029296875, + "KL/rejected_KL_mean": -901.1683959960938, + "KL/std": 343.0787353515625, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0020415354520082474, + "fcm_dpo/delta": -0.16515761613845825, + "fcm_dpo/margin": 276.330810546875, + "fcm_dpo/q_t": 0.3766445517539978, + "grad_norm": 41.392757415771484, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": 0.6525043845176697, + "logits/rejected": 0.6880553960800171, + "logps/chosen": -692.67822265625, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1011.1080322265625, + "loss": 1.0149, + "margin_dpo/margin_mean": 276.330810546875, + "margin_dpo/margin_std": 346.77313232421875, + "step": 557 + }, + { + "KL/chosen_KL_mean": -586.452392578125, + "KL/mean": -775.2528686523438, + "KL/rejected_KL_mean": -964.0532836914062, + "KL/std": 351.22686767578125, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.002029248047620058, + "fcm_dpo/delta": -0.3690915107727051, + "fcm_dpo/margin": 377.600830078125, + "fcm_dpo/q_t": 0.33377692103385925, + "grad_norm": 43.24068069458008, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": 0.6908626556396484, + "logits/rejected": 0.7192879915237427, + "logps/chosen": -648.8206787109375, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1066.21435546875, + "loss": 0.871, + "margin_dpo/margin_mean": 377.6008605957031, + "margin_dpo/margin_std": 354.3226318359375, + "step": 558 + }, + { + "KL/chosen_KL_mean": -637.966552734375, + "KL/mean": -789.5410766601562, + "KL/rejected_KL_mean": -941.1155395507812, + "KL/std": 333.1988830566406, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.002018560655415058, + "fcm_dpo/delta": -0.213303804397583, + "fcm_dpo/margin": 303.1488952636719, + "fcm_dpo/q_t": 0.3657890558242798, + "grad_norm": 48.709434509277344, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": 0.8149224519729614, + "logits/rejected": 0.8686637282371521, + "logps/chosen": -698.7189331054688, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1034.557861328125, + "loss": 0.9623, + "margin_dpo/margin_mean": 303.14892578125, + "margin_dpo/margin_std": 329.3605651855469, + "step": 559 + }, + { + "KL/chosen_KL_mean": -571.0098266601562, + "KL/mean": -679.856201171875, + "KL/rejected_KL_mean": -788.7025146484375, + "KL/std": 323.24468994140625, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0020151352509856224, + "fcm_dpo/delta": -0.03888411819934845, + "fcm_dpo/margin": 217.69271850585938, + "fcm_dpo/q_t": 0.39956405758857727, + "grad_norm": 55.629764556884766, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": 0.4465806484222412, + "logits/rejected": 0.4723140001296997, + "logps/chosen": -629.1136474609375, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -868.6937255859375, + "loss": 1.0638, + "margin_dpo/margin_mean": 217.69271850585938, + "margin_dpo/margin_std": 269.05999755859375, + "step": 560 + }, + { + "KL/chosen_KL_mean": -673.965087890625, + "KL/mean": -787.7200317382812, + "KL/rejected_KL_mean": -901.4749755859375, + "KL/std": 304.8691711425781, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0020142877474427223, + "fcm_dpo/delta": -0.05854104459285736, + "fcm_dpo/margin": 227.50987243652344, + "fcm_dpo/q_t": 0.3960952162742615, + "grad_norm": 51.882137298583984, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": 0.7760847806930542, + "logits/rejected": 0.7906183004379272, + "logps/chosen": -724.8779296875, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1000.5435180664062, + "loss": 1.0734, + "margin_dpo/margin_mean": 227.50987243652344, + "margin_dpo/margin_std": 317.88116455078125, + "step": 561 + }, + { + "KL/chosen_KL_mean": -642.1142578125, + "KL/mean": -790.0842895507812, + "KL/rejected_KL_mean": -938.0543212890625, + "KL/std": 346.1695861816406, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0020061535760760307, + "fcm_dpo/delta": -0.19500666856765747, + "fcm_dpo/margin": 295.94012451171875, + "fcm_dpo/q_t": 0.3703376352787018, + "grad_norm": 59.679080963134766, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": 0.7268211841583252, + "logits/rejected": 0.7541632652282715, + "logps/chosen": -701.5786743164062, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1034.5970458984375, + "loss": 0.9774, + "margin_dpo/margin_mean": 295.94012451171875, + "margin_dpo/margin_std": 337.890625, + "step": 562 + }, + { + "KL/chosen_KL_mean": -687.55078125, + "KL/mean": -797.836181640625, + "KL/rejected_KL_mean": -908.12158203125, + "KL/std": 320.2120361328125, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0020053801126778126, + "fcm_dpo/delta": -0.042591825127601624, + "fcm_dpo/margin": 220.57081604003906, + "fcm_dpo/q_t": 0.3998965620994568, + "grad_norm": 57.87691879272461, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": 0.8005231618881226, + "logits/rejected": 0.8062916994094849, + "logps/chosen": -748.158935546875, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1002.6892700195312, + "loss": 1.1082, + "margin_dpo/margin_mean": 220.57083129882812, + "margin_dpo/margin_std": 343.8956604003906, + "step": 563 + }, + { + "KL/chosen_KL_mean": -634.7228393554688, + "KL/mean": -771.0428466796875, + "KL/rejected_KL_mean": -907.3629150390625, + "KL/std": 327.91192626953125, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0019993302412331104, + "fcm_dpo/delta": -0.14587946236133575, + "fcm_dpo/margin": 272.6400451660156, + "fcm_dpo/q_t": 0.3766234517097473, + "grad_norm": 38.11875915527344, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": 0.7042189836502075, + "logits/rejected": 0.7536500692367554, + "logps/chosen": -708.9912109375, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1000.60107421875, + "loss": 0.9902, + "margin_dpo/margin_mean": 272.6400451660156, + "margin_dpo/margin_std": 289.1048583984375, + "step": 564 + }, + { + "KL/chosen_KL_mean": -632.6318359375, + "KL/mean": -782.6513061523438, + "KL/rejected_KL_mean": -932.6707763671875, + "KL/std": 380.77520751953125, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0019912496209144592, + "fcm_dpo/delta": -0.19869673252105713, + "fcm_dpo/margin": 300.0389709472656, + "fcm_dpo/q_t": 0.3763428330421448, + "grad_norm": 56.01233673095703, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": 0.5935712456703186, + "logits/rejected": 0.6219205856323242, + "logps/chosen": -701.6517333984375, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1018.44970703125, + "loss": 1.0108, + "margin_dpo/margin_mean": 300.0389709472656, + "margin_dpo/margin_std": 404.12225341796875, + "step": 565 + }, + { + "KL/chosen_KL_mean": -642.1557006835938, + "KL/mean": -796.1903076171875, + "KL/rejected_KL_mean": -950.224853515625, + "KL/std": 374.41680908203125, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.001984806265681982, + "fcm_dpo/delta": -0.21270835399627686, + "fcm_dpo/margin": 308.0691223144531, + "fcm_dpo/q_t": 0.37035882472991943, + "grad_norm": 45.875240325927734, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": 0.6599386930465698, + "logits/rejected": 0.6761844158172607, + "logps/chosen": -708.7010498046875, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1054.09423828125, + "loss": 1.0119, + "margin_dpo/margin_mean": 308.0691223144531, + "margin_dpo/margin_std": 402.3214111328125, + "step": 566 + }, + { + "KL/chosen_KL_mean": -652.52880859375, + "KL/mean": -776.4134521484375, + "KL/rejected_KL_mean": -900.298095703125, + "KL/std": 291.2756652832031, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0019785696640610695, + "fcm_dpo/delta": -0.09067464619874954, + "fcm_dpo/margin": 247.7694091796875, + "fcm_dpo/q_t": 0.38949573040008545, + "grad_norm": 47.885589599609375, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": 0.8914766311645508, + "logits/rejected": 0.8936726450920105, + "logps/chosen": -705.3870849609375, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -985.6690673828125, + "loss": 1.044, + "margin_dpo/margin_mean": 247.7694091796875, + "margin_dpo/margin_std": 316.2713317871094, + "step": 567 + }, + { + "KL/chosen_KL_mean": -598.94775390625, + "KL/mean": -765.7769775390625, + "KL/rejected_KL_mean": -932.6060791015625, + "KL/std": 323.48638916015625, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0019696527160704136, + "fcm_dpo/delta": -0.25895076990127563, + "fcm_dpo/margin": 333.65838623046875, + "fcm_dpo/q_t": 0.35337409377098083, + "grad_norm": 46.58582305908203, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": 0.7595484256744385, + "logits/rejected": 0.7833364009857178, + "logps/chosen": -644.14013671875, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1021.698486328125, + "loss": 0.9176, + "margin_dpo/margin_mean": 333.65838623046875, + "margin_dpo/margin_std": 311.866943359375, + "step": 568 + }, + { + "KL/chosen_KL_mean": -666.6150512695312, + "KL/mean": -815.2042846679688, + "KL/rejected_KL_mean": -963.7935791015625, + "KL/std": 370.8099365234375, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.001962947426363826, + "fcm_dpo/delta": -0.18437910079956055, + "fcm_dpo/margin": 297.1784362792969, + "fcm_dpo/q_t": 0.3724650740623474, + "grad_norm": 71.04789733886719, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": 0.8110783100128174, + "logits/rejected": 0.843557596206665, + "logps/chosen": -730.3356323242188, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1042.896728515625, + "loss": 1.002, + "margin_dpo/margin_mean": 297.17840576171875, + "margin_dpo/margin_std": 367.439453125, + "step": 569 + }, + { + "KL/chosen_KL_mean": -638.09375, + "KL/mean": -764.7444458007812, + "KL/rejected_KL_mean": -891.3950805664062, + "KL/std": 320.95025634765625, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0019569010473787785, + "fcm_dpo/delta": -0.09620112180709839, + "fcm_dpo/margin": 253.30133056640625, + "fcm_dpo/q_t": 0.3894237279891968, + "grad_norm": 37.809024810791016, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": 0.78208988904953, + "logits/rejected": 0.8008078336715698, + "logps/chosen": -699.708251953125, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -973.5369873046875, + "loss": 1.04, + "margin_dpo/margin_mean": 253.3013458251953, + "margin_dpo/margin_std": 315.4437255859375, + "step": 570 + }, + { + "KL/chosen_KL_mean": -662.4198608398438, + "KL/mean": -811.1021728515625, + "KL/rejected_KL_mean": -959.784423828125, + "KL/std": 329.9828796386719, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0019500839989632368, + "fcm_dpo/delta": -0.18109729886054993, + "fcm_dpo/margin": 297.3645324707031, + "fcm_dpo/q_t": 0.3739006519317627, + "grad_norm": 39.92853927612305, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": 0.7382972240447998, + "logits/rejected": 0.7426289319992065, + "logps/chosen": -715.4739379882812, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1051.1212158203125, + "loss": 1.0085, + "margin_dpo/margin_mean": 297.364501953125, + "margin_dpo/margin_std": 373.27911376953125, + "step": 571 + }, + { + "KL/chosen_KL_mean": -678.1715087890625, + "KL/mean": -833.0279541015625, + "KL/rejected_KL_mean": -987.8843994140625, + "KL/std": 377.7936096191406, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0019430238753557205, + "fcm_dpo/delta": -0.2030588537454605, + "fcm_dpo/margin": 309.712890625, + "fcm_dpo/q_t": 0.3744911551475525, + "grad_norm": 42.70563888549805, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": 0.8517757654190063, + "logits/rejected": 0.8819972276687622, + "logps/chosen": -726.6307983398438, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1071.44140625, + "loss": 1.0104, + "margin_dpo/margin_mean": 309.712890625, + "margin_dpo/margin_std": 415.9700622558594, + "step": 572 + }, + { + "KL/chosen_KL_mean": -667.1253662109375, + "KL/mean": -805.055419921875, + "KL/rejected_KL_mean": -942.9853515625, + "KL/std": 328.956787109375, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0019386674975976348, + "fcm_dpo/delta": -0.1355137974023819, + "fcm_dpo/margin": 275.8599853515625, + "fcm_dpo/q_t": 0.38182532787323, + "grad_norm": 50.55799102783203, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": 0.8703145384788513, + "logits/rejected": 0.8950026035308838, + "logps/chosen": -718.748046875, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1028.310302734375, + "loss": 1.0117, + "margin_dpo/margin_mean": 275.8599853515625, + "margin_dpo/margin_std": 323.5845947265625, + "step": 573 + }, + { + "KL/chosen_KL_mean": -623.9051513671875, + "KL/mean": -775.93994140625, + "KL/rejected_KL_mean": -927.9746704101562, + "KL/std": 338.5635986328125, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0019311457872390747, + "fcm_dpo/delta": -0.18828019499778748, + "fcm_dpo/margin": 304.06951904296875, + "fcm_dpo/q_t": 0.37160220742225647, + "grad_norm": 31.432661056518555, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": 0.7182271480560303, + "logits/rejected": 0.7304551601409912, + "logps/chosen": -674.9495849609375, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1020.7810668945312, + "loss": 0.9838, + "margin_dpo/margin_mean": 304.06951904296875, + "margin_dpo/margin_std": 351.0728759765625, + "step": 574 + }, + { + "KL/chosen_KL_mean": -689.9169921875, + "KL/mean": -828.30908203125, + "KL/rejected_KL_mean": -966.701171875, + "KL/std": 323.4555969238281, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0019232071936130524, + "fcm_dpo/delta": -0.13334991037845612, + "fcm_dpo/margin": 276.7842712402344, + "fcm_dpo/q_t": 0.38222748041152954, + "grad_norm": 37.91510009765625, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": 0.6665240526199341, + "logits/rejected": 0.6760618686676025, + "logps/chosen": -761.7070922851562, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1062.08740234375, + "loss": 1.0511, + "margin_dpo/margin_mean": 276.7842712402344, + "margin_dpo/margin_std": 375.59991455078125, + "step": 575 + }, + { + "KL/chosen_KL_mean": -672.4468994140625, + "KL/mean": -805.0631103515625, + "KL/rejected_KL_mean": -937.6793823242188, + "KL/std": 327.9583740234375, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0019194602500647306, + "fcm_dpo/delta": -0.10980869084596634, + "fcm_dpo/margin": 265.2325439453125, + "fcm_dpo/q_t": 0.38800713419914246, + "grad_norm": 47.09749984741211, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": 0.8165856599807739, + "logits/rejected": 0.8254492282867432, + "logps/chosen": -726.7098388671875, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1038.43359375, + "loss": 1.054, + "margin_dpo/margin_mean": 265.2325439453125, + "margin_dpo/margin_std": 362.41400146484375, + "step": 576 + }, + { + "KL/chosen_KL_mean": -615.9920654296875, + "KL/mean": -751.357177734375, + "KL/rejected_KL_mean": -886.72216796875, + "KL/std": 333.6733703613281, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0019160760566592216, + "fcm_dpo/delta": -0.11936530470848083, + "fcm_dpo/margin": 270.73016357421875, + "fcm_dpo/q_t": 0.38337188959121704, + "grad_norm": 43.75292205810547, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": 0.6726531386375427, + "logits/rejected": 0.6918753385543823, + "logps/chosen": -677.9017944335938, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -970.7929077148438, + "loss": 1.0254, + "margin_dpo/margin_mean": 270.73016357421875, + "margin_dpo/margin_std": 319.0467834472656, + "step": 577 + }, + { + "KL/chosen_KL_mean": -591.515869140625, + "KL/mean": -738.7362060546875, + "KL/rejected_KL_mean": -885.95654296875, + "KL/std": 323.5951843261719, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.0019089055713266134, + "fcm_dpo/delta": -0.16321313381195068, + "fcm_dpo/margin": 294.4405822753906, + "fcm_dpo/q_t": 0.37309831380844116, + "grad_norm": 34.028411865234375, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": 0.6372066736221313, + "logits/rejected": 0.666854977607727, + "logps/chosen": -640.7796020507812, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -969.3927612304688, + "loss": 0.974, + "margin_dpo/margin_mean": 294.4405822753906, + "margin_dpo/margin_std": 288.65966796875, + "step": 578 + }, + { + "KL/chosen_KL_mean": -642.06396484375, + "KL/mean": -747.1798095703125, + "KL/rejected_KL_mean": -852.2955322265625, + "KL/std": 320.43572998046875, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0019085835665464401, + "fcm_dpo/delta": -0.001251152716577053, + "fcm_dpo/margin": 210.23155212402344, + "fcm_dpo/q_t": 0.4067249000072479, + "grad_norm": 39.71281051635742, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": 0.724726676940918, + "logits/rejected": 0.7190146446228027, + "logps/chosen": -700.9498291015625, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -947.0831909179688, + "loss": 1.1068, + "margin_dpo/margin_mean": 210.23155212402344, + "margin_dpo/margin_std": 306.0252990722656, + "step": 579 + }, + { + "KL/chosen_KL_mean": -525.185302734375, + "KL/mean": -689.303466796875, + "KL/rejected_KL_mean": -853.4217529296875, + "KL/std": 359.1301574707031, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.001900829141959548, + "fcm_dpo/delta": -0.22562053799629211, + "fcm_dpo/margin": 328.2363586425781, + "fcm_dpo/q_t": 0.36515653133392334, + "grad_norm": 37.448734283447266, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.514417290687561, + "logits/rejected": 0.5336433053016663, + "logps/chosen": -573.8921508789062, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -935.1801147460938, + "loss": 0.962, + "margin_dpo/margin_mean": 328.2363586425781, + "margin_dpo/margin_std": 366.62811279296875, + "step": 580 + }, + { + "KL/chosen_KL_mean": -655.191650390625, + "KL/mean": -778.04345703125, + "KL/rejected_KL_mean": -900.895263671875, + "KL/std": 337.1972961425781, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.001898544142022729, + "fcm_dpo/delta": -0.06678511202335358, + "fcm_dpo/margin": 245.70358276367188, + "fcm_dpo/q_t": 0.39739686250686646, + "grad_norm": 49.07291793823242, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": 0.7495021820068359, + "logits/rejected": 0.7691392302513123, + "logps/chosen": -717.4140625, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -992.6309814453125, + "loss": 1.069, + "margin_dpo/margin_mean": 245.70358276367188, + "margin_dpo/margin_std": 346.63946533203125, + "step": 581 + }, + { + "KL/chosen_KL_mean": -601.930419921875, + "KL/mean": -746.531494140625, + "KL/rejected_KL_mean": -891.1324462890625, + "KL/std": 314.1972351074219, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0018948864890262485, + "fcm_dpo/delta": -0.14882872998714447, + "fcm_dpo/margin": 289.2019958496094, + "fcm_dpo/q_t": 0.3772484064102173, + "grad_norm": 41.60464096069336, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": 0.6935842037200928, + "logits/rejected": 0.6898167133331299, + "logps/chosen": -660.1590576171875, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1001.2020263671875, + "loss": 1.0077, + "margin_dpo/margin_mean": 289.2020263671875, + "margin_dpo/margin_std": 332.71429443359375, + "step": 582 + }, + { + "KL/chosen_KL_mean": -637.0928955078125, + "KL/mean": -742.5768432617188, + "KL/rejected_KL_mean": -848.060791015625, + "KL/std": 341.53253173828125, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0018882699077948928, + "fcm_dpo/delta": -0.09783103317022324, + "fcm_dpo/margin": 210.9678192138672, + "fcm_dpo/q_t": 0.4121342897415161, + "grad_norm": 42.65382766723633, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": 0.659672200679779, + "logits/rejected": 0.6845858097076416, + "logps/chosen": -693.955810546875, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -936.4647216796875, + "loss": 1.1317, + "margin_dpo/margin_mean": 210.96783447265625, + "margin_dpo/margin_std": 342.4588623046875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -580.7406005859375, + "KL/mean": -754.2426147460938, + "KL/rejected_KL_mean": -927.7447509765625, + "KL/std": 353.34283447265625, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0018810718320310116, + "fcm_dpo/delta": -0.2544022500514984, + "fcm_dpo/margin": 347.00408935546875, + "fcm_dpo/q_t": 0.35696494579315186, + "grad_norm": 35.105098724365234, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": 0.6956484317779541, + "logits/rejected": 0.7299377918243408, + "logps/chosen": -637.6412963867188, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1025.3807373046875, + "loss": 0.9307, + "margin_dpo/margin_mean": 347.00408935546875, + "margin_dpo/margin_std": 349.63427734375, + "step": 584 + }, + { + "KL/chosen_KL_mean": -645.4781494140625, + "KL/mean": -802.0462036132812, + "KL/rejected_KL_mean": -958.6141357421875, + "KL/std": 353.531494140625, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0018733416218310595, + "fcm_dpo/delta": -0.18771910667419434, + "fcm_dpo/margin": 313.13604736328125, + "fcm_dpo/q_t": 0.37193310260772705, + "grad_norm": 42.06230545043945, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": 0.7249537110328674, + "logits/rejected": 0.7784085273742676, + "logps/chosen": -706.4521484375, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1042.78369140625, + "loss": 0.9907, + "margin_dpo/margin_mean": 313.13604736328125, + "margin_dpo/margin_std": 370.82611083984375, + "step": 585 + }, + { + "KL/chosen_KL_mean": -675.3883666992188, + "KL/mean": -795.0338134765625, + "KL/rejected_KL_mean": -914.6793212890625, + "KL/std": 306.4168701171875, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0018702002707868814, + "fcm_dpo/delta": -0.047738946974277496, + "fcm_dpo/margin": 239.2908935546875, + "fcm_dpo/q_t": 0.3991108238697052, + "grad_norm": 33.37091827392578, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": 0.8224821090698242, + "logits/rejected": 0.8405784368515015, + "logps/chosen": -732.2439575195312, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1006.48193359375, + "loss": 1.0946, + "margin_dpo/margin_mean": 239.2908935546875, + "margin_dpo/margin_std": 363.67486572265625, + "step": 586 + }, + { + "KL/chosen_KL_mean": -560.426025390625, + "KL/mean": -703.6683959960938, + "KL/rejected_KL_mean": -846.9107666015625, + "KL/std": 361.57440185546875, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.001866683829575777, + "fcm_dpo/delta": -0.13547834753990173, + "fcm_dpo/margin": 286.48480224609375, + "fcm_dpo/q_t": 0.3800261616706848, + "grad_norm": 35.93736267089844, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": 0.8114941120147705, + "logits/rejected": 0.8189507126808167, + "logps/chosen": -605.1175537109375, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -929.53466796875, + "loss": 1.0104, + "margin_dpo/margin_mean": 286.48480224609375, + "margin_dpo/margin_std": 334.61212158203125, + "step": 587 + }, + { + "KL/chosen_KL_mean": -664.2178344726562, + "KL/mean": -787.646728515625, + "KL/rejected_KL_mean": -911.0755615234375, + "KL/std": 307.1063232421875, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0018624973017722368, + "fcm_dpo/delta": -0.06009062007069588, + "fcm_dpo/margin": 246.85772705078125, + "fcm_dpo/q_t": 0.39537313580513, + "grad_norm": 36.63571548461914, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": 0.756576657295227, + "logits/rejected": 0.757891058921814, + "logps/chosen": -714.5128173828125, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1018.4454345703125, + "loss": 1.0545, + "margin_dpo/margin_mean": 246.85775756835938, + "margin_dpo/margin_std": 307.301513671875, + "step": 588 + }, + { + "KL/chosen_KL_mean": -622.6163330078125, + "KL/mean": -796.9681396484375, + "KL/rejected_KL_mean": -971.31982421875, + "KL/std": 362.0784606933594, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.001855609123595059, + "fcm_dpo/delta": -0.24866998195648193, + "fcm_dpo/margin": 348.7034912109375, + "fcm_dpo/q_t": 0.35633811354637146, + "grad_norm": 32.16123580932617, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": 0.68644118309021, + "logits/rejected": 0.7073366641998291, + "logps/chosen": -682.5462646484375, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1082.97509765625, + "loss": 0.9283, + "margin_dpo/margin_mean": 348.7034912109375, + "margin_dpo/margin_std": 338.5122985839844, + "step": 589 + }, + { + "KL/chosen_KL_mean": -605.8663330078125, + "KL/mean": -746.03515625, + "KL/rejected_KL_mean": -886.2040405273438, + "KL/std": 306.12921142578125, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.00184974679723382, + "fcm_dpo/delta": -0.11919307708740234, + "fcm_dpo/margin": 280.33770751953125, + "fcm_dpo/q_t": 0.3826559782028198, + "grad_norm": 31.531644821166992, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": 0.6296848654747009, + "logits/rejected": 0.6299532055854797, + "logps/chosen": -661.6761474609375, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -992.266845703125, + "loss": 1.013, + "margin_dpo/margin_mean": 280.33770751953125, + "margin_dpo/margin_std": 317.9443664550781, + "step": 590 + }, + { + "KL/chosen_KL_mean": -658.3350830078125, + "KL/mean": -800.257080078125, + "KL/rejected_KL_mean": -942.1790771484375, + "KL/std": 317.8763427734375, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0018456528196111321, + "fcm_dpo/delta": -0.1245189979672432, + "fcm_dpo/margin": 283.84393310546875, + "fcm_dpo/q_t": 0.38119828701019287, + "grad_norm": 39.50395584106445, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": 0.8882416486740112, + "logits/rejected": 0.9287067651748657, + "logps/chosen": -714.57568359375, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1025.96533203125, + "loss": 1.0146, + "margin_dpo/margin_mean": 283.84393310546875, + "margin_dpo/margin_std": 329.9251403808594, + "step": 591 + }, + { + "KL/chosen_KL_mean": -684.3240356445312, + "KL/mean": -814.406494140625, + "KL/rejected_KL_mean": -944.4888916015625, + "KL/std": 319.9940185546875, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0018418513936921954, + "fcm_dpo/delta": -0.07958149164915085, + "fcm_dpo/margin": 260.1648864746094, + "fcm_dpo/q_t": 0.3939093053340912, + "grad_norm": 54.47821044921875, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": 0.9103589057922363, + "logits/rejected": 0.9275103211402893, + "logps/chosen": -732.2642822265625, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1020.2218017578125, + "loss": 1.1035, + "margin_dpo/margin_mean": 260.1648864746094, + "margin_dpo/margin_std": 419.576904296875, + "step": 592 + }, + { + "KL/chosen_KL_mean": -637.0269775390625, + "KL/mean": -739.2296142578125, + "KL/rejected_KL_mean": -841.4321899414062, + "KL/std": 360.16485595703125, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0018416817765682936, + "fcm_dpo/delta": 0.023636892437934875, + "fcm_dpo/margin": 204.4052734375, + "fcm_dpo/q_t": 0.416016548871994, + "grad_norm": 50.98042297363281, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": 0.6217687129974365, + "logits/rejected": 0.6403443813323975, + "logps/chosen": -685.7177124023438, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -902.3402099609375, + "loss": 1.1386, + "margin_dpo/margin_mean": 204.4052734375, + "margin_dpo/margin_std": 345.10260009765625, + "step": 593 + }, + { + "KL/chosen_KL_mean": -619.81982421875, + "KL/mean": -756.1556396484375, + "KL/rejected_KL_mean": -892.4913940429688, + "KL/std": 398.40509033203125, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.0018413348589092493, + "fcm_dpo/delta": -0.10270573943853378, + "fcm_dpo/margin": 272.67156982421875, + "fcm_dpo/q_t": 0.39058220386505127, + "grad_norm": 35.35045623779297, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": 0.697027325630188, + "logits/rejected": 0.7250281572341919, + "logps/chosen": -674.7546997070312, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -978.591064453125, + "loss": 1.0658, + "margin_dpo/margin_mean": 272.67156982421875, + "margin_dpo/margin_std": 378.8766784667969, + "step": 594 + }, + { + "KL/chosen_KL_mean": -627.3757934570312, + "KL/mean": -767.450927734375, + "KL/rejected_KL_mean": -907.5260620117188, + "KL/std": 316.260009765625, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.001834713970310986, + "fcm_dpo/delta": -0.11464294046163559, + "fcm_dpo/margin": 280.1502380371094, + "fcm_dpo/q_t": 0.3867257535457611, + "grad_norm": 46.561920166015625, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": 0.739148736000061, + "logits/rejected": 0.7639570236206055, + "logps/chosen": -676.7962036132812, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -988.1533813476562, + "loss": 1.0239, + "margin_dpo/margin_mean": 280.1502380371094, + "margin_dpo/margin_std": 342.6709899902344, + "step": 595 + }, + { + "KL/chosen_KL_mean": -662.783203125, + "KL/mean": -775.5308837890625, + "KL/rejected_KL_mean": -888.278564453125, + "KL/std": 343.95355224609375, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0018342589028179646, + "fcm_dpo/delta": -0.013679193332791328, + "fcm_dpo/margin": 225.49537658691406, + "fcm_dpo/q_t": 0.4073355197906494, + "grad_norm": 77.3415298461914, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": 0.7206860184669495, + "logits/rejected": 0.7380715608596802, + "logps/chosen": -722.5748901367188, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -968.3696899414062, + "loss": 1.1059, + "margin_dpo/margin_mean": 225.49539184570312, + "margin_dpo/margin_std": 343.763427734375, + "step": 596 + }, + { + "KL/chosen_KL_mean": -646.1209106445312, + "KL/mean": -807.0723876953125, + "KL/rejected_KL_mean": -968.02392578125, + "KL/std": 381.4713134765625, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.001826832420192659, + "fcm_dpo/delta": -0.18953149020671844, + "fcm_dpo/margin": 321.9029541015625, + "fcm_dpo/q_t": 0.37355172634124756, + "grad_norm": 34.25974655151367, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": 0.6096208691596985, + "logits/rejected": 0.6200867295265198, + "logps/chosen": -703.3817138671875, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1068.7176513671875, + "loss": 1.0064, + "margin_dpo/margin_mean": 321.9029541015625, + "margin_dpo/margin_std": 406.32745361328125, + "step": 597 + }, + { + "KL/chosen_KL_mean": -664.310302734375, + "KL/mean": -794.2281494140625, + "KL/rejected_KL_mean": -924.1460571289062, + "KL/std": 315.14752197265625, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0018232379807159305, + "fcm_dpo/delta": -0.07430021464824677, + "fcm_dpo/margin": 259.83575439453125, + "fcm_dpo/q_t": 0.39087483286857605, + "grad_norm": 37.37078857421875, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": 0.8423196077346802, + "logits/rejected": 0.8699223399162292, + "logps/chosen": -716.8287963867188, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1013.5899047851562, + "loss": 1.0668, + "margin_dpo/margin_mean": 259.83575439453125, + "margin_dpo/margin_std": 344.1370849609375, + "step": 598 + }, + { + "KL/chosen_KL_mean": -626.2774047851562, + "KL/mean": -763.4021606445312, + "KL/rejected_KL_mean": -900.52685546875, + "KL/std": 346.244873046875, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0018202085047960281, + "fcm_dpo/delta": -0.09983537346124649, + "fcm_dpo/margin": 274.24951171875, + "fcm_dpo/q_t": 0.38408952951431274, + "grad_norm": 41.77424621582031, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": 0.7416328191757202, + "logits/rejected": 0.7566128969192505, + "logps/chosen": -676.080078125, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -983.50537109375, + "loss": 1.0111, + "margin_dpo/margin_mean": 274.24951171875, + "margin_dpo/margin_std": 277.67083740234375, + "step": 599 + }, + { + "KL/chosen_KL_mean": -705.58984375, + "KL/mean": -824.9664306640625, + "KL/rejected_KL_mean": -944.3430786132812, + "KL/std": 326.93121337890625, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0018192067509517074, + "fcm_dpo/delta": -0.034503381699323654, + "fcm_dpo/margin": 238.75323486328125, + "fcm_dpo/q_t": 0.40190309286117554, + "grad_norm": 48.88149642944336, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": 0.804427981376648, + "logits/rejected": 0.8255845308303833, + "logps/chosen": -772.024658203125, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1029.799560546875, + "loss": 1.0904, + "margin_dpo/margin_mean": 238.75323486328125, + "margin_dpo/margin_std": 345.59619140625, + "step": 600 + }, + { + "KL/chosen_KL_mean": -692.0425415039062, + "KL/mean": -840.3756103515625, + "KL/rejected_KL_mean": -988.7086791992188, + "KL/std": 345.9216003417969, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0018148890230804682, + "fcm_dpo/delta": -0.1392229199409485, + "fcm_dpo/margin": 296.6661376953125, + "fcm_dpo/q_t": 0.37792307138442993, + "grad_norm": 33.74208450317383, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": 0.7667839527130127, + "logits/rejected": 0.7968571186065674, + "logps/chosen": -751.1761474609375, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1083.399658203125, + "loss": 0.9988, + "margin_dpo/margin_mean": 296.6661376953125, + "margin_dpo/margin_std": 326.7383117675781, + "step": 601 + }, + { + "KL/chosen_KL_mean": -555.3302001953125, + "KL/mean": -723.0023193359375, + "KL/rejected_KL_mean": -890.6744384765625, + "KL/std": 331.974853515625, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.00180962891317904, + "fcm_dpo/delta": -0.20805974304676056, + "fcm_dpo/margin": 335.3442687988281, + "fcm_dpo/q_t": 0.36284053325653076, + "grad_norm": 59.749237060546875, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": 0.7236630320549011, + "logits/rejected": 0.7578376531600952, + "logps/chosen": -603.9237060546875, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -978.343017578125, + "loss": 0.9435, + "margin_dpo/margin_mean": 335.3442687988281, + "margin_dpo/margin_std": 311.5479736328125, + "step": 602 + }, + { + "KL/chosen_KL_mean": -609.5101318359375, + "KL/mean": -759.4876708984375, + "KL/rejected_KL_mean": -909.465087890625, + "KL/std": 391.6522216796875, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0018031394574791193, + "fcm_dpo/delta": -0.1416066586971283, + "fcm_dpo/margin": 299.9549560546875, + "fcm_dpo/q_t": 0.38634228706359863, + "grad_norm": 42.65598678588867, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": 0.6419580578804016, + "logits/rejected": 0.6755662560462952, + "logps/chosen": -679.9248046875, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1009.7907104492188, + "loss": 1.0323, + "margin_dpo/margin_mean": 299.9549560546875, + "margin_dpo/margin_std": 421.4832458496094, + "step": 603 + }, + { + "KL/chosen_KL_mean": -600.9806518554688, + "KL/mean": -767.5335693359375, + "KL/rejected_KL_mean": -934.08642578125, + "KL/std": 326.75537109375, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0017961736302822828, + "fcm_dpo/delta": -0.19950509071350098, + "fcm_dpo/margin": 333.10589599609375, + "fcm_dpo/q_t": 0.3668884038925171, + "grad_norm": 33.88639450073242, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": 0.7511519193649292, + "logits/rejected": 0.7747603058815002, + "logps/chosen": -647.438720703125, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1025.94091796875, + "loss": 0.9677, + "margin_dpo/margin_mean": 333.1058654785156, + "margin_dpo/margin_std": 355.6172180175781, + "step": 604 + }, + { + "KL/chosen_KL_mean": -659.4561767578125, + "KL/mean": -810.7955322265625, + "KL/rejected_KL_mean": -962.1348876953125, + "KL/std": 340.7897644042969, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0017896464560180902, + "fcm_dpo/delta": -0.14257404208183289, + "fcm_dpo/margin": 302.67877197265625, + "fcm_dpo/q_t": 0.38029053807258606, + "grad_norm": 42.83294677734375, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": 0.6616764068603516, + "logits/rejected": 0.6807286739349365, + "logps/chosen": -725.7054443359375, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1064.43994140625, + "loss": 1.0075, + "margin_dpo/margin_mean": 302.67877197265625, + "margin_dpo/margin_std": 363.27703857421875, + "step": 605 + }, + { + "KL/chosen_KL_mean": -648.4679565429688, + "KL/mean": -794.330810546875, + "KL/rejected_KL_mean": -940.1937255859375, + "KL/std": 342.36376953125, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0017870450392365456, + "fcm_dpo/delta": -0.12196290493011475, + "fcm_dpo/margin": 291.72576904296875, + "fcm_dpo/q_t": 0.3850659728050232, + "grad_norm": 30.897106170654297, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": 0.7438620328903198, + "logits/rejected": 0.7460161447525024, + "logps/chosen": -703.2870483398438, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1038.565185546875, + "loss": 1.0246, + "margin_dpo/margin_mean": 291.72576904296875, + "margin_dpo/margin_std": 360.50091552734375, + "step": 606 + }, + { + "KL/chosen_KL_mean": -669.443603515625, + "KL/mean": -786.2998046875, + "KL/rejected_KL_mean": -903.1561279296875, + "KL/std": 335.10894775390625, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0017845355905592442, + "fcm_dpo/delta": -0.017139529809355736, + "fcm_dpo/margin": 233.71246337890625, + "fcm_dpo/q_t": 0.40500515699386597, + "grad_norm": 45.48642349243164, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": 0.6589515209197998, + "logits/rejected": 0.6749454736709595, + "logps/chosen": -727.527587890625, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -982.9332275390625, + "loss": 1.0844, + "margin_dpo/margin_mean": 233.7124786376953, + "margin_dpo/margin_std": 309.8780517578125, + "step": 607 + }, + { + "KL/chosen_KL_mean": -637.2819213867188, + "KL/mean": -772.918212890625, + "KL/rejected_KL_mean": -908.5545043945312, + "KL/std": 325.33880615234375, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0017822610680013895, + "fcm_dpo/delta": -0.0838763415813446, + "fcm_dpo/margin": 271.2725830078125, + "fcm_dpo/q_t": 0.38851040601730347, + "grad_norm": 33.986629486083984, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": 0.6684036254882812, + "logits/rejected": 0.710300087928772, + "logps/chosen": -694.7327880859375, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1003.327880859375, + "loss": 1.0187, + "margin_dpo/margin_mean": 271.2725830078125, + "margin_dpo/margin_std": 280.986572265625, + "step": 608 + }, + { + "KL/chosen_KL_mean": -640.9236450195312, + "KL/mean": -804.48828125, + "KL/rejected_KL_mean": -968.0528564453125, + "KL/std": 381.065673828125, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.0017765960656106472, + "fcm_dpo/delta": -0.1822184920310974, + "fcm_dpo/margin": 327.1292419433594, + "fcm_dpo/q_t": 0.37069469690322876, + "grad_norm": 40.187992095947266, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": 0.7760666608810425, + "logits/rejected": 0.8133994340896606, + "logps/chosen": -699.72900390625, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1056.868896484375, + "loss": 0.9758, + "margin_dpo/margin_mean": 327.1292419433594, + "margin_dpo/margin_std": 363.18603515625, + "step": 609 + }, + { + "KL/chosen_KL_mean": -637.43408203125, + "KL/mean": -776.3323974609375, + "KL/rejected_KL_mean": -915.2307739257812, + "KL/std": 343.41064453125, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.001772496267221868, + "fcm_dpo/delta": -0.09284333884716034, + "fcm_dpo/margin": 277.796630859375, + "fcm_dpo/q_t": 0.3895209729671478, + "grad_norm": 44.13329315185547, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": 0.6482257843017578, + "logits/rejected": 0.6769185066223145, + "logps/chosen": -703.129150390625, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -998.6361694335938, + "loss": 1.0432, + "margin_dpo/margin_mean": 277.796630859375, + "margin_dpo/margin_std": 358.1323547363281, + "step": 610 + }, + { + "KL/chosen_KL_mean": -669.404296875, + "KL/mean": -826.623779296875, + "KL/rejected_KL_mean": -983.84326171875, + "KL/std": 360.76446533203125, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0017680858727544546, + "fcm_dpo/delta": -0.15679652988910675, + "fcm_dpo/margin": 314.43896484375, + "fcm_dpo/q_t": 0.37949225306510925, + "grad_norm": 39.941650390625, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": 0.7833685874938965, + "logits/rejected": 0.8231631517410278, + "logps/chosen": -722.0037841796875, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1070.17431640625, + "loss": 1.0185, + "margin_dpo/margin_mean": 314.43896484375, + "margin_dpo/margin_std": 402.5318298339844, + "step": 611 + }, + { + "KL/chosen_KL_mean": -671.4500122070312, + "KL/mean": -835.6005859375, + "KL/rejected_KL_mean": -999.7510375976562, + "KL/std": 325.54937744140625, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0017613675445318222, + "fcm_dpo/delta": -0.17928098142147064, + "fcm_dpo/margin": 328.30108642578125, + "fcm_dpo/q_t": 0.36846381425857544, + "grad_norm": 33.99043273925781, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": 0.823289692401886, + "logits/rejected": 0.8644186854362488, + "logps/chosen": -730.7737426757812, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1088.0634765625, + "loss": 0.958, + "margin_dpo/margin_mean": 328.3010559082031, + "margin_dpo/margin_std": 311.77532958984375, + "step": 612 + }, + { + "KL/chosen_KL_mean": -623.370361328125, + "KL/mean": -793.8212280273438, + "KL/rejected_KL_mean": -964.2720947265625, + "KL/std": 374.82122802734375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0017542322166264057, + "fcm_dpo/delta": -0.1992303431034088, + "fcm_dpo/margin": 340.90179443359375, + "fcm_dpo/q_t": 0.3657914400100708, + "grad_norm": 40.224212646484375, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": 0.6576024293899536, + "logits/rejected": 0.6783339977264404, + "logps/chosen": -683.1002807617188, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1069.379638671875, + "loss": 0.95, + "margin_dpo/margin_mean": 340.9018249511719, + "margin_dpo/margin_std": 332.4084167480469, + "step": 613 + }, + { + "KL/chosen_KL_mean": -681.5625, + "KL/mean": -856.3742065429688, + "KL/rejected_KL_mean": -1031.18603515625, + "KL/std": 389.60107421875, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0017475431086495519, + "fcm_dpo/delta": -0.21223589777946472, + "fcm_dpo/margin": 349.62347412109375, + "fcm_dpo/q_t": 0.3665129542350769, + "grad_norm": 64.0534439086914, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.8236353397369385, + "logits/rejected": 0.8569149374961853, + "logps/chosen": -734.50146484375, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1135.8653564453125, + "loss": 0.9781, + "margin_dpo/margin_mean": 349.62347412109375, + "margin_dpo/margin_std": 406.82904052734375, + "step": 614 + }, + { + "KL/chosen_KL_mean": -664.225830078125, + "KL/mean": -799.5118408203125, + "KL/rejected_KL_mean": -934.7978515625, + "KL/std": 416.8905029296875, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0017442090902477503, + "fcm_dpo/delta": -0.07229100167751312, + "fcm_dpo/margin": 270.572021484375, + "fcm_dpo/q_t": 0.39797767996788025, + "grad_norm": 38.44405746459961, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": 0.5803609490394592, + "logits/rejected": 0.6051238775253296, + "logps/chosen": -730.0430908203125, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1029.975341796875, + "loss": 1.0683, + "margin_dpo/margin_mean": 270.5719909667969, + "margin_dpo/margin_std": 387.41741943359375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -766.855224609375, + "KL/mean": -877.464111328125, + "KL/rejected_KL_mean": -988.0728759765625, + "KL/std": 328.676513671875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0017435417976230383, + "fcm_dpo/delta": 0.014333507046103477, + "fcm_dpo/margin": 221.21768188476562, + "fcm_dpo/q_t": 0.4137648046016693, + "grad_norm": 32.51485061645508, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": 0.8107097148895264, + "logits/rejected": 0.841254711151123, + "logps/chosen": -831.9880981445312, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1062.7734375, + "loss": 1.1236, + "margin_dpo/margin_mean": 221.21768188476562, + "margin_dpo/margin_std": 351.0145263671875, + "step": 616 + }, + { + "KL/chosen_KL_mean": -714.6624755859375, + "KL/mean": -800.6077880859375, + "KL/rejected_KL_mean": -886.5531616210938, + "KL/std": 291.61138916015625, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.00174618826713413, + "fcm_dpo/delta": 0.10011502355337143, + "fcm_dpo/margin": 171.89059448242188, + "fcm_dpo/q_t": 0.4304659366607666, + "grad_norm": 52.887245178222656, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": 0.6625223159790039, + "logits/rejected": 0.6671267151832581, + "logps/chosen": -777.6680908203125, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -950.7872314453125, + "loss": 1.171, + "margin_dpo/margin_mean": 171.8905792236328, + "margin_dpo/margin_std": 293.72247314453125, + "step": 617 + }, + { + "KL/chosen_KL_mean": -724.234130859375, + "KL/mean": -902.2966918945312, + "KL/rejected_KL_mean": -1080.3592529296875, + "KL/std": 390.39910888671875, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0017421364318579435, + "fcm_dpo/delta": -0.22173896431922913, + "fcm_dpo/margin": 356.1251220703125, + "fcm_dpo/q_t": 0.3659171462059021, + "grad_norm": 39.62374496459961, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": 0.7291539311408997, + "logits/rejected": 0.8004436492919922, + "logps/chosen": -791.33544921875, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1172.5126953125, + "loss": 0.9913, + "margin_dpo/margin_mean": 356.1251220703125, + "margin_dpo/margin_std": 441.4393310546875, + "step": 618 + }, + { + "KL/chosen_KL_mean": -728.648193359375, + "KL/mean": -861.4984130859375, + "KL/rejected_KL_mean": -994.3486328125, + "KL/std": 370.9404296875, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.001737086451612413, + "fcm_dpo/delta": -0.061843037605285645, + "fcm_dpo/margin": 265.70050048828125, + "fcm_dpo/q_t": 0.4029574394226074, + "grad_norm": 38.175376892089844, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": 0.7683799266815186, + "logits/rejected": 0.7990607023239136, + "logps/chosen": -784.62646484375, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1087.5340576171875, + "loss": 1.111, + "margin_dpo/margin_mean": 265.70050048828125, + "margin_dpo/margin_std": 447.9293518066406, + "step": 619 + }, + { + "KL/chosen_KL_mean": -672.473388671875, + "KL/mean": -815.234130859375, + "KL/rejected_KL_mean": -957.9948120117188, + "KL/std": 341.2877197265625, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0017334980657324195, + "fcm_dpo/delta": -0.09551216661930084, + "fcm_dpo/margin": 285.5214538574219, + "fcm_dpo/q_t": 0.3908153772354126, + "grad_norm": 32.01239776611328, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": 0.785138726234436, + "logits/rejected": 0.8241643905639648, + "logps/chosen": -732.2708740234375, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1036.405517578125, + "loss": 1.0439, + "margin_dpo/margin_mean": 285.5214538574219, + "margin_dpo/margin_std": 368.9264221191406, + "step": 620 + }, + { + "KL/chosen_KL_mean": -687.373046875, + "KL/mean": -868.0811767578125, + "KL/rejected_KL_mean": -1048.7894287109375, + "KL/std": 371.1351623535156, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.001728828763589263, + "fcm_dpo/delta": -0.22618059813976288, + "fcm_dpo/margin": 361.4163818359375, + "fcm_dpo/q_t": 0.3613079786300659, + "grad_norm": 38.20238494873047, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": 0.9200645685195923, + "logits/rejected": 0.95698082447052, + "logps/chosen": -741.3067626953125, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1137.158935546875, + "loss": 0.9513, + "margin_dpo/margin_mean": 361.4163818359375, + "margin_dpo/margin_std": 372.3457336425781, + "step": 621 + }, + { + "KL/chosen_KL_mean": -665.9921264648438, + "KL/mean": -806.6402587890625, + "KL/rejected_KL_mean": -947.2883911132812, + "KL/std": 302.46826171875, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.001722477376461029, + "fcm_dpo/delta": -0.08505165576934814, + "fcm_dpo/margin": 281.2962646484375, + "fcm_dpo/q_t": 0.3908979296684265, + "grad_norm": 36.63888931274414, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": 0.6427109241485596, + "logits/rejected": 0.6497113704681396, + "logps/chosen": -726.2779541015625, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1032.80712890625, + "loss": 1.035, + "margin_dpo/margin_mean": 281.2962646484375, + "margin_dpo/margin_std": 336.60589599609375, + "step": 622 + }, + { + "KL/chosen_KL_mean": -711.294921875, + "KL/mean": -825.9144287109375, + "KL/rejected_KL_mean": -940.533935546875, + "KL/std": 309.9722900390625, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0017228268552571535, + "fcm_dpo/delta": 0.005081073381006718, + "fcm_dpo/margin": 229.23895263671875, + "fcm_dpo/q_t": 0.41226062178611755, + "grad_norm": 47.433170318603516, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": 0.9321935772895813, + "logits/rejected": 0.9691169261932373, + "logps/chosen": -775.451904296875, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1025.616943359375, + "loss": 1.1197, + "margin_dpo/margin_mean": 229.2389678955078, + "margin_dpo/margin_std": 367.1021728515625, + "step": 623 + }, + { + "KL/chosen_KL_mean": -696.296142578125, + "KL/mean": -863.6477661132812, + "KL/rejected_KL_mean": -1030.9993896484375, + "KL/std": 356.6759338378906, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0017184499884024262, + "fcm_dpo/delta": -0.17617523670196533, + "fcm_dpo/margin": 334.7032165527344, + "fcm_dpo/q_t": 0.3690810203552246, + "grad_norm": 42.57719421386719, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": 0.7041852474212646, + "logits/rejected": 0.7567317485809326, + "logps/chosen": -768.2147827148438, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1128.13134765625, + "loss": 0.974, + "margin_dpo/margin_mean": 334.7032165527344, + "margin_dpo/margin_std": 348.50762939453125, + "step": 624 + }, + { + "KL/chosen_KL_mean": -664.9523315429688, + "KL/mean": -823.4305419921875, + "KL/rejected_KL_mean": -981.90869140625, + "KL/std": 354.4139404296875, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.001713288715109229, + "fcm_dpo/delta": -0.14380808174610138, + "fcm_dpo/margin": 316.95635986328125, + "fcm_dpo/q_t": 0.37743937969207764, + "grad_norm": 47.25299835205078, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": 0.7686718106269836, + "logits/rejected": 0.8225444555282593, + "logps/chosen": -723.29443359375, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1067.9990234375, + "loss": 0.9844, + "margin_dpo/margin_mean": 316.95635986328125, + "margin_dpo/margin_std": 327.14141845703125, + "step": 625 + }, + { + "KL/chosen_KL_mean": -745.0408935546875, + "KL/mean": -854.3203125, + "KL/rejected_KL_mean": -963.599609375, + "KL/std": 376.24139404296875, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.001712932251393795, + "fcm_dpo/delta": 0.02570553869009018, + "fcm_dpo/margin": 218.5587158203125, + "fcm_dpo/q_t": 0.4149106740951538, + "grad_norm": 38.309783935546875, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": 0.5565162897109985, + "logits/rejected": 0.5711402297019958, + "logps/chosen": -820.153564453125, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1062.788330078125, + "loss": 1.1276, + "margin_dpo/margin_mean": 218.5587158203125, + "margin_dpo/margin_std": 336.4215087890625, + "step": 626 + }, + { + "KL/chosen_KL_mean": -641.1722412109375, + "KL/mean": -817.5615234375, + "KL/rejected_KL_mean": -993.9508056640625, + "KL/std": 406.2264404296875, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0017082680715247989, + "fcm_dpo/delta": -0.20381920039653778, + "fcm_dpo/margin": 352.77862548828125, + "fcm_dpo/q_t": 0.3748325705528259, + "grad_norm": 42.84407043457031, + "learning_rate": 9.897955805412e-09, + "logits/chosen": 0.6335510015487671, + "logits/rejected": 0.6553667187690735, + "logps/chosen": -688.9154052734375, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1100.705322265625, + "loss": 0.9995, + "margin_dpo/margin_mean": 352.77862548828125, + "margin_dpo/margin_std": 457.0096740722656, + "step": 627 + }, + { + "KL/chosen_KL_mean": -719.5697021484375, + "KL/mean": -873.6804809570312, + "KL/rejected_KL_mean": -1027.7913818359375, + "KL/std": 348.00006103515625, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0017023030668497086, + "fcm_dpo/delta": -0.1253746598958969, + "fcm_dpo/margin": 308.2216796875, + "fcm_dpo/q_t": 0.38220924139022827, + "grad_norm": 30.413835525512695, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": 0.7931725978851318, + "logits/rejected": 0.8421310186386108, + "logps/chosen": -779.7525634765625, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1129.345947265625, + "loss": 1.0143, + "margin_dpo/margin_mean": 308.2217102050781, + "margin_dpo/margin_std": 359.5828857421875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -730.75341796875, + "KL/mean": -879.44677734375, + "KL/rejected_KL_mean": -1028.14013671875, + "KL/std": 343.5316162109375, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0016987058334052563, + "fcm_dpo/delta": -0.10571747273206711, + "fcm_dpo/margin": 297.38671875, + "fcm_dpo/q_t": 0.3835851550102234, + "grad_norm": 35.71873474121094, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": 0.816564679145813, + "logits/rejected": 0.849432110786438, + "logps/chosen": -794.9669799804688, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1119.7938232421875, + "loss": 1.009, + "margin_dpo/margin_mean": 297.38671875, + "margin_dpo/margin_std": 309.06085205078125, + "step": 629 + }, + { + "KL/chosen_KL_mean": -654.0045776367188, + "KL/mean": -750.141845703125, + "KL/rejected_KL_mean": -846.2791137695312, + "KL/std": 400.1324462890625, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.001700415974482894, + "fcm_dpo/delta": 0.07324282079935074, + "fcm_dpo/margin": 192.2745361328125, + "fcm_dpo/q_t": 0.42894867062568665, + "grad_norm": 54.908294677734375, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": 0.5927219390869141, + "logits/rejected": 0.6017290353775024, + "logps/chosen": -713.2955932617188, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -929.87744140625, + "loss": 1.1853, + "margin_dpo/margin_mean": 192.2745361328125, + "margin_dpo/margin_std": 392.1331787109375, + "step": 630 + }, + { + "KL/chosen_KL_mean": -764.03466796875, + "KL/mean": -888.670654296875, + "KL/rejected_KL_mean": -1013.3065795898438, + "KL/std": 371.807861328125, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.001700051361694932, + "fcm_dpo/delta": -0.02387644723057747, + "fcm_dpo/margin": 249.27194213867188, + "fcm_dpo/q_t": 0.40643563866615295, + "grad_norm": 45.844459533691406, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": 0.9363858699798584, + "logits/rejected": 0.9753478169441223, + "logps/chosen": -823.48828125, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1094.2581787109375, + "loss": 1.1261, + "margin_dpo/margin_mean": 249.27195739746094, + "margin_dpo/margin_std": 427.6947937011719, + "step": 631 + }, + { + "KL/chosen_KL_mean": -725.7098388671875, + "KL/mean": -872.8812255859375, + "KL/rejected_KL_mean": -1020.0526733398438, + "KL/std": 327.9253234863281, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.0016959484200924635, + "fcm_dpo/delta": -0.09984620660543442, + "fcm_dpo/margin": 294.3428039550781, + "fcm_dpo/q_t": 0.38834017515182495, + "grad_norm": 45.349552154541016, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": 0.9171468019485474, + "logits/rejected": 0.9603859186172485, + "logps/chosen": -787.0614013671875, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1106.212890625, + "loss": 1.0516, + "margin_dpo/margin_mean": 294.3428039550781, + "margin_dpo/margin_std": 387.4168701171875, + "step": 632 + }, + { + "KL/chosen_KL_mean": -720.7661743164062, + "KL/mean": -831.4991455078125, + "KL/rejected_KL_mean": -942.232177734375, + "KL/std": 358.87078857421875, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0016980583313852549, + "fcm_dpo/delta": 0.02390975132584572, + "fcm_dpo/margin": 221.46600341796875, + "fcm_dpo/q_t": 0.4131929874420166, + "grad_norm": 53.301876068115234, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": 0.8496856689453125, + "logits/rejected": 0.8542599678039551, + "logps/chosen": -778.0443115234375, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1033.816162109375, + "loss": 1.1251, + "margin_dpo/margin_mean": 221.4660186767578, + "margin_dpo/margin_std": 332.71551513671875, + "step": 633 + }, + { + "KL/chosen_KL_mean": -792.3712158203125, + "KL/mean": -901.6676635742188, + "KL/rejected_KL_mean": -1010.964111328125, + "KL/std": 454.35882568359375, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.001697540283203125, + "fcm_dpo/delta": 0.029037578031420708, + "fcm_dpo/margin": 218.59278869628906, + "fcm_dpo/q_t": 0.42003846168518066, + "grad_norm": 52.335445404052734, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": 0.6731219291687012, + "logits/rejected": 0.6735209226608276, + "logps/chosen": -858.990234375, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1118.0897216796875, + "loss": 1.1518, + "margin_dpo/margin_mean": 218.59280395507812, + "margin_dpo/margin_std": 401.4050598144531, + "step": 634 + }, + { + "KL/chosen_KL_mean": -741.116455078125, + "KL/mean": -864.07470703125, + "KL/rejected_KL_mean": -987.032958984375, + "KL/std": 404.74505615234375, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.0016973521560430527, + "fcm_dpo/delta": -0.01748630218207836, + "fcm_dpo/margin": 245.91641235351562, + "fcm_dpo/q_t": 0.40735745429992676, + "grad_norm": 34.814083099365234, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": 0.70428866147995, + "logits/rejected": 0.7689028978347778, + "logps/chosen": -815.072021484375, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1069.533447265625, + "loss": 1.0997, + "margin_dpo/margin_mean": 245.9164276123047, + "margin_dpo/margin_std": 367.5152282714844, + "step": 635 + }, + { + "KL/chosen_KL_mean": -703.9051513671875, + "KL/mean": -855.1896362304688, + "KL/rejected_KL_mean": -1006.47412109375, + "KL/std": 428.46051025390625, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0016955379396677017, + "fcm_dpo/delta": -0.11360122263431549, + "fcm_dpo/margin": 302.5690002441406, + "fcm_dpo/q_t": 0.38569992780685425, + "grad_norm": 37.48571014404297, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": 0.658734142780304, + "logits/rejected": 0.7012739181518555, + "logps/chosen": -763.5340576171875, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1088.452880859375, + "loss": 1.0143, + "margin_dpo/margin_mean": 302.5690002441406, + "margin_dpo/margin_std": 347.57843017578125, + "step": 636 + }, + { + "KL/chosen_KL_mean": -737.3985595703125, + "KL/mean": -899.4334716796875, + "KL/rejected_KL_mean": -1061.468505859375, + "KL/std": 417.96282958984375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0016889767721295357, + "fcm_dpo/delta": -0.14828452467918396, + "fcm_dpo/margin": 324.0699462890625, + "fcm_dpo/q_t": 0.3803471326828003, + "grad_norm": 37.131507873535156, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": 0.9481520652770996, + "logits/rejected": 0.9857852458953857, + "logps/chosen": -787.05126953125, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1159.87353515625, + "loss": 1.012, + "margin_dpo/margin_mean": 324.0699462890625, + "margin_dpo/margin_std": 394.3244323730469, + "step": 637 + }, + { + "KL/chosen_KL_mean": -688.5628662109375, + "KL/mean": -819.9420776367188, + "KL/rejected_KL_mean": -951.3212890625, + "KL/std": 348.21624755859375, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0016891320701688528, + "fcm_dpo/delta": -0.04418157786130905, + "fcm_dpo/margin": 262.7584533691406, + "fcm_dpo/q_t": 0.3968521058559418, + "grad_norm": 45.23640060424805, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": 0.7256067991256714, + "logits/rejected": 0.7496404647827148, + "logps/chosen": -746.719482421875, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1030.622802734375, + "loss": 1.0838, + "margin_dpo/margin_mean": 262.7584228515625, + "margin_dpo/margin_std": 362.1710205078125, + "step": 638 + }, + { + "KL/chosen_KL_mean": -853.5523681640625, + "KL/mean": -937.1101684570312, + "KL/rejected_KL_mean": -1020.6680297851562, + "KL/std": 349.890869140625, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.001690305769443512, + "fcm_dpo/delta": 0.11783594638109207, + "fcm_dpo/margin": 167.11573791503906, + "fcm_dpo/q_t": 0.43763846158981323, + "grad_norm": 88.3899154663086, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": 0.6721237897872925, + "logits/rejected": 0.6904313564300537, + "logps/chosen": -925.87548828125, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1094.94287109375, + "loss": 1.2248, + "margin_dpo/margin_mean": 167.11572265625, + "margin_dpo/margin_std": 386.36669921875, + "step": 639 + }, + { + "KL/chosen_KL_mean": -694.4031372070312, + "KL/mean": -895.0210571289062, + "KL/rejected_KL_mean": -1095.63916015625, + "KL/std": 400.9637145996094, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0016848583472892642, + "fcm_dpo/delta": -0.2778169512748718, + "fcm_dpo/margin": 401.23583984375, + "fcm_dpo/q_t": 0.35703223943710327, + "grad_norm": 47.618370056152344, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": 0.6385558843612671, + "logits/rejected": 0.6884221434593201, + "logps/chosen": -750.5374755859375, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1204.2392578125, + "loss": 0.9413, + "margin_dpo/margin_mean": 401.23583984375, + "margin_dpo/margin_std": 451.4275207519531, + "step": 640 + }, + { + "KL/chosen_KL_mean": -811.0089721679688, + "KL/mean": -929.4209594726562, + "KL/rejected_KL_mean": -1047.8330078125, + "KL/std": 335.8045654296875, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0016812297981232405, + "fcm_dpo/delta": 0.001833716407418251, + "fcm_dpo/margin": 236.82395935058594, + "fcm_dpo/q_t": 0.41201651096343994, + "grad_norm": 53.26057052612305, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": 0.8148948550224304, + "logits/rejected": 0.8487507104873657, + "logps/chosen": -876.005859375, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1134.8251953125, + "loss": 1.1382, + "margin_dpo/margin_mean": 236.82394409179688, + "margin_dpo/margin_std": 415.1270751953125, + "step": 641 + }, + { + "KL/chosen_KL_mean": -744.359130859375, + "KL/mean": -895.4037475585938, + "KL/rejected_KL_mean": -1046.4482421875, + "KL/std": 433.0734558105469, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0016799044096842408, + "fcm_dpo/delta": -0.10802680999040604, + "fcm_dpo/margin": 302.0892333984375, + "fcm_dpo/q_t": 0.39718884229660034, + "grad_norm": 44.460121154785156, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": 0.6550302505493164, + "logits/rejected": 0.6758990287780762, + "logps/chosen": -810.04833984375, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1156.6904296875, + "loss": 1.0827, + "margin_dpo/margin_mean": 302.0892333984375, + "margin_dpo/margin_std": 499.2666931152344, + "step": 642 + }, + { + "KL/chosen_KL_mean": -702.412841796875, + "KL/mean": -835.7662353515625, + "KL/rejected_KL_mean": -969.1197509765625, + "KL/std": 366.89251708984375, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0016758517595008016, + "fcm_dpo/delta": -0.04728736728429794, + "fcm_dpo/margin": 266.7069091796875, + "fcm_dpo/q_t": 0.39776095747947693, + "grad_norm": 34.05390167236328, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": 0.8631190061569214, + "logits/rejected": 0.8654353618621826, + "logps/chosen": -754.36279296875, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1056.588134765625, + "loss": 1.0555, + "margin_dpo/margin_mean": 266.7069091796875, + "margin_dpo/margin_std": 312.92120361328125, + "step": 643 + }, + { + "KL/chosen_KL_mean": -694.9741821289062, + "KL/mean": -823.258056640625, + "KL/rejected_KL_mean": -951.5419921875, + "KL/std": 399.345703125, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0016758383717387915, + "fcm_dpo/delta": -0.030094806104898453, + "fcm_dpo/margin": 256.5677795410156, + "fcm_dpo/q_t": 0.40442702174186707, + "grad_norm": 45.29292297363281, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": 0.6767255067825317, + "logits/rejected": 0.6989917755126953, + "logps/chosen": -753.9918212890625, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1038.6787109375, + "loss": 1.0992, + "margin_dpo/margin_mean": 256.5677795410156, + "margin_dpo/margin_std": 390.051025390625, + "step": 644 + }, + { + "KL/chosen_KL_mean": -719.2545776367188, + "KL/mean": -799.374755859375, + "KL/rejected_KL_mean": -879.494873046875, + "KL/std": 339.6253356933594, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0016760069411247969, + "fcm_dpo/delta": 0.01409115269780159, + "fcm_dpo/margin": 160.2403106689453, + "fcm_dpo/q_t": 0.44135618209838867, + "grad_norm": 61.2418327331543, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": 0.6548548340797424, + "logits/rejected": 0.6372078061103821, + "logps/chosen": -775.130615234375, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -977.2756958007812, + "loss": 1.2392, + "margin_dpo/margin_mean": 160.2403106689453, + "margin_dpo/margin_std": 391.553466796875, + "step": 645 + }, + { + "KL/chosen_KL_mean": -721.5513916015625, + "KL/mean": -840.5370483398438, + "KL/rejected_KL_mean": -959.522705078125, + "KL/std": 296.7061462402344, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0016753734089434147, + "fcm_dpo/delta": 0.0012831278145313263, + "fcm_dpo/margin": 237.97140502929688, + "fcm_dpo/q_t": 0.41020840406417847, + "grad_norm": 51.653343200683594, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": 0.8203037977218628, + "logits/rejected": 0.8329168558120728, + "logps/chosen": -782.8271484375, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1037.028564453125, + "loss": 1.1107, + "margin_dpo/margin_mean": 237.97140502929688, + "margin_dpo/margin_std": 353.1919250488281, + "step": 646 + }, + { + "KL/chosen_KL_mean": -660.61962890625, + "KL/mean": -793.9033203125, + "KL/rejected_KL_mean": -927.18701171875, + "KL/std": 388.52874755859375, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0016752383671700954, + "fcm_dpo/delta": -0.04677200689911842, + "fcm_dpo/margin": 266.5673522949219, + "fcm_dpo/q_t": 0.3991919159889221, + "grad_norm": 35.03009796142578, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": 0.641678512096405, + "logits/rejected": 0.6610992550849915, + "logps/chosen": -715.4720458984375, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1020.7064208984375, + "loss": 1.0679, + "margin_dpo/margin_mean": 266.5673828125, + "margin_dpo/margin_std": 346.222900390625, + "step": 647 + }, + { + "KL/chosen_KL_mean": -677.3397827148438, + "KL/mean": -869.4111328125, + "KL/rejected_KL_mean": -1061.482666015625, + "KL/std": 412.6510925292969, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.001668265787884593, + "fcm_dpo/delta": -0.24242860078811646, + "fcm_dpo/margin": 384.142822265625, + "fcm_dpo/q_t": 0.3667376637458801, + "grad_norm": 31.67422103881836, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.8339884281158447, + "logits/rejected": 0.8820208311080933, + "logps/chosen": -731.51123046875, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1160.1954345703125, + "loss": 0.9766, + "margin_dpo/margin_mean": 384.142822265625, + "margin_dpo/margin_std": 486.6613464355469, + "step": 648 + }, + { + "KL/chosen_KL_mean": -716.1796875, + "KL/mean": -810.433349609375, + "KL/rejected_KL_mean": -904.6871337890625, + "KL/std": 323.3931884765625, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0016689208569005132, + "fcm_dpo/delta": 0.08563382178544998, + "fcm_dpo/margin": 188.50747680664062, + "fcm_dpo/q_t": 0.43161576986312866, + "grad_norm": 47.553104400634766, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": 0.8203299045562744, + "logits/rejected": 0.8447185754776001, + "logps/chosen": -778.6600341796875, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -984.7643432617188, + "loss": 1.191, + "margin_dpo/margin_mean": 188.50746154785156, + "margin_dpo/margin_std": 377.98187255859375, + "step": 649 + }, + { + "KL/chosen_KL_mean": -707.0875244140625, + "KL/mean": -863.1333618164062, + "KL/rejected_KL_mean": -1019.17919921875, + "KL/std": 377.8154296875, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0016653584316372871, + "fcm_dpo/delta": -0.12048791348934174, + "fcm_dpo/margin": 312.09173583984375, + "fcm_dpo/q_t": 0.38619866967201233, + "grad_norm": 29.04220199584961, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": 0.7828747034072876, + "logits/rejected": 0.8022740483283997, + "logps/chosen": -763.1802978515625, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1117.444091796875, + "loss": 1.0357, + "margin_dpo/margin_mean": 312.09173583984375, + "margin_dpo/margin_std": 403.83441162109375, + "step": 650 + }, + { + "KL/chosen_KL_mean": -593.96484375, + "KL/mean": -782.5360107421875, + "KL/rejected_KL_mean": -971.1072387695312, + "KL/std": 391.57916259765625, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0016592828324064612, + "fcm_dpo/delta": -0.22725170850753784, + "fcm_dpo/margin": 377.14239501953125, + "fcm_dpo/q_t": 0.3595596253871918, + "grad_norm": 63.38020324707031, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": 0.7632284164428711, + "logits/rejected": 0.7778958082199097, + "logps/chosen": -637.3902587890625, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1071.065185546875, + "loss": 0.9291, + "margin_dpo/margin_mean": 377.14239501953125, + "margin_dpo/margin_std": 346.1991271972656, + "step": 651 + }, + { + "KL/chosen_KL_mean": -671.4737548828125, + "KL/mean": -807.6138916015625, + "KL/rejected_KL_mean": -943.7540283203125, + "KL/std": 384.1431579589844, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.001656344160437584, + "fcm_dpo/delta": -0.05122281610965729, + "fcm_dpo/margin": 272.28033447265625, + "fcm_dpo/q_t": 0.39961695671081543, + "grad_norm": 40.14388656616211, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": 0.737112283706665, + "logits/rejected": 0.7612301111221313, + "logps/chosen": -734.050537109375, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1055.5218505859375, + "loss": 1.0712, + "margin_dpo/margin_mean": 272.2803039550781, + "margin_dpo/margin_std": 375.20404052734375, + "step": 652 + }, + { + "KL/chosen_KL_mean": -743.8433227539062, + "KL/mean": -883.25341796875, + "KL/rejected_KL_mean": -1022.66357421875, + "KL/std": 411.49359130859375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0016542873345315456, + "fcm_dpo/delta": -0.061540331691503525, + "fcm_dpo/margin": 278.8201904296875, + "fcm_dpo/q_t": 0.3968043923377991, + "grad_norm": 38.853851318359375, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": 0.7948933839797974, + "logits/rejected": 0.8079038262367249, + "logps/chosen": -804.956298828125, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1125.9130859375, + "loss": 1.0651, + "margin_dpo/margin_mean": 278.8201904296875, + "margin_dpo/margin_std": 375.02972412109375, + "step": 653 + }, + { + "KL/chosen_KL_mean": -726.544677734375, + "KL/mean": -848.15966796875, + "KL/rejected_KL_mean": -969.774658203125, + "KL/std": 343.7318115234375, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0016536302864551544, + "fcm_dpo/delta": -0.0022262753918766975, + "fcm_dpo/margin": 243.2299346923828, + "fcm_dpo/q_t": 0.41062667965888977, + "grad_norm": 42.08026123046875, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": 0.7836779356002808, + "logits/rejected": 0.8090945482254028, + "logps/chosen": -788.2728271484375, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1068.5484619140625, + "loss": 1.1194, + "margin_dpo/margin_mean": 243.22991943359375, + "margin_dpo/margin_std": 389.2502136230469, + "step": 654 + }, + { + "KL/chosen_KL_mean": -679.081787109375, + "KL/mean": -817.4296875, + "KL/rejected_KL_mean": -955.7776489257812, + "KL/std": 348.0972900390625, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0016519902274012566, + "fcm_dpo/delta": -0.057391420006752014, + "fcm_dpo/margin": 276.69573974609375, + "fcm_dpo/q_t": 0.3998359441757202, + "grad_norm": 36.78379821777344, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": 0.8706467151641846, + "logits/rejected": 0.889626681804657, + "logps/chosen": -728.6586303710938, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1054.0694580078125, + "loss": 1.0733, + "margin_dpo/margin_mean": 276.69573974609375, + "margin_dpo/margin_std": 389.52410888671875, + "step": 655 + }, + { + "KL/chosen_KL_mean": -690.5958251953125, + "KL/mean": -912.99169921875, + "KL/rejected_KL_mean": -1135.3876953125, + "KL/std": 422.8914794921875, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0016444651409983635, + "fcm_dpo/delta": -0.33377766609191895, + "fcm_dpo/margin": 444.79180908203125, + "fcm_dpo/q_t": 0.34086811542510986, + "grad_norm": 63.93142318725586, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": 0.7137954235076904, + "logits/rejected": 0.7652997374534607, + "logps/chosen": -743.145263671875, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1249.062255859375, + "loss": 0.8747, + "margin_dpo/margin_mean": 444.79180908203125, + "margin_dpo/margin_std": 397.85107421875, + "step": 656 + }, + { + "KL/chosen_KL_mean": -690.6363525390625, + "KL/mean": -861.8397216796875, + "KL/rejected_KL_mean": -1033.04296875, + "KL/std": 392.8434753417969, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0016366320196539164, + "fcm_dpo/delta": -0.161345973610878, + "fcm_dpo/margin": 342.4067077636719, + "fcm_dpo/q_t": 0.375383198261261, + "grad_norm": 40.89299011230469, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": 0.8126430511474609, + "logits/rejected": 0.8293417692184448, + "logps/chosen": -737.3368530273438, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1130.9578857421875, + "loss": 0.9833, + "margin_dpo/margin_mean": 342.4067077636719, + "margin_dpo/margin_std": 372.02740478515625, + "step": 657 + }, + { + "KL/chosen_KL_mean": -736.629150390625, + "KL/mean": -884.2093505859375, + "KL/rejected_KL_mean": -1031.7896728515625, + "KL/std": 336.26202392578125, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0016331966035068035, + "fcm_dpo/delta": -0.0824928730726242, + "fcm_dpo/margin": 295.16046142578125, + "fcm_dpo/q_t": 0.3892222046852112, + "grad_norm": 35.538124084472656, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": 0.8119245767593384, + "logits/rejected": 0.843580961227417, + "logps/chosen": -797.5873413085938, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1127.7291259765625, + "loss": 1.027, + "margin_dpo/margin_mean": 295.16046142578125, + "margin_dpo/margin_std": 326.5550537109375, + "step": 658 + }, + { + "KL/chosen_KL_mean": -659.392333984375, + "KL/mean": -816.2928466796875, + "KL/rejected_KL_mean": -973.193359375, + "KL/std": 354.91510009765625, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0016295635141432285, + "fcm_dpo/delta": -0.11201423406600952, + "fcm_dpo/margin": 313.801025390625, + "fcm_dpo/q_t": 0.38446879386901855, + "grad_norm": 31.847055435180664, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": 0.5909181833267212, + "logits/rejected": 0.6381077766418457, + "logps/chosen": -736.1353759765625, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1060.664306640625, + "loss": 1.0137, + "margin_dpo/margin_mean": 313.8009948730469, + "margin_dpo/margin_std": 349.9155578613281, + "step": 659 + }, + { + "KL/chosen_KL_mean": -720.464111328125, + "KL/mean": -887.29736328125, + "KL/rejected_KL_mean": -1054.130615234375, + "KL/std": 357.30816650390625, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0016261215787380934, + "fcm_dpo/delta": -0.1433412730693817, + "fcm_dpo/margin": 333.6664123535156, + "fcm_dpo/q_t": 0.3779456317424774, + "grad_norm": 33.365447998046875, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": 0.9184255599975586, + "logits/rejected": 0.9564040899276733, + "logps/chosen": -779.511962890625, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1130.090576171875, + "loss": 1.0006, + "margin_dpo/margin_mean": 333.66644287109375, + "margin_dpo/margin_std": 377.02362060546875, + "step": 660 + }, + { + "KL/chosen_KL_mean": -638.8114013671875, + "KL/mean": -796.2860717773438, + "KL/rejected_KL_mean": -953.7607421875, + "KL/std": 377.16326904296875, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0016229778993874788, + "fcm_dpo/delta": -0.11173608154058456, + "fcm_dpo/margin": 314.9493408203125, + "fcm_dpo/q_t": 0.3850606381893158, + "grad_norm": 35.247310638427734, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": 0.7841619253158569, + "logits/rejected": 0.8210753798484802, + "logps/chosen": -689.4853515625, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1039.7664794921875, + "loss": 1.016, + "margin_dpo/margin_mean": 314.9493408203125, + "margin_dpo/margin_std": 354.48577880859375, + "step": 661 + }, + { + "KL/chosen_KL_mean": -687.7066040039062, + "KL/mean": -818.9132690429688, + "KL/rejected_KL_mean": -950.1199951171875, + "KL/std": 350.72735595703125, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0016206044238060713, + "fcm_dpo/delta": -0.025375759229063988, + "fcm_dpo/margin": 262.4133605957031, + "fcm_dpo/q_t": 0.4023483991622925, + "grad_norm": 29.283893585205078, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": 0.5866655707359314, + "logits/rejected": 0.6208598613739014, + "logps/chosen": -756.9676513671875, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1039.1759033203125, + "loss": 1.0888, + "margin_dpo/margin_mean": 262.4133605957031, + "margin_dpo/margin_std": 369.4774169921875, + "step": 662 + }, + { + "KL/chosen_KL_mean": -656.1820068359375, + "KL/mean": -812.25830078125, + "KL/rejected_KL_mean": -968.3346557617188, + "KL/std": 422.9426574707031, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0016177725046873093, + "fcm_dpo/delta": -0.10551868379116058, + "fcm_dpo/margin": 312.15264892578125, + "fcm_dpo/q_t": 0.3918311297893524, + "grad_norm": 45.034820556640625, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": 0.6390251517295837, + "logits/rejected": 0.6553741693496704, + "logps/chosen": -721.0609130859375, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1082.260009765625, + "loss": 1.0505, + "margin_dpo/margin_mean": 312.1526794433594, + "margin_dpo/margin_std": 459.21331787109375, + "step": 663 + }, + { + "KL/chosen_KL_mean": -716.7635498046875, + "KL/mean": -887.5372314453125, + "KL/rejected_KL_mean": -1058.31103515625, + "KL/std": 370.28521728515625, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.001612539286725223, + "fcm_dpo/delta": -0.15167748928070068, + "fcm_dpo/margin": 341.5474853515625, + "fcm_dpo/q_t": 0.3753560781478882, + "grad_norm": 39.01245880126953, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": 0.7999619245529175, + "logits/rejected": 0.844018816947937, + "logps/chosen": -777.6519775390625, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1163.832763671875, + "loss": 0.9959, + "margin_dpo/margin_mean": 341.5474853515625, + "margin_dpo/margin_std": 379.9185791015625, + "step": 664 + }, + { + "KL/chosen_KL_mean": -668.1253662109375, + "KL/mean": -827.755859375, + "KL/rejected_KL_mean": -987.3864135742188, + "KL/std": 340.3009338378906, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0016089812852442265, + "fcm_dpo/delta": -0.11428647488355637, + "fcm_dpo/margin": 319.26104736328125, + "fcm_dpo/q_t": 0.3840535581111908, + "grad_norm": 29.758516311645508, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": 0.9919731616973877, + "logits/rejected": 1.045011281967163, + "logps/chosen": -728.6895141601562, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1072.1953125, + "loss": 1.0079, + "margin_dpo/margin_mean": 319.26104736328125, + "margin_dpo/margin_std": 344.6383056640625, + "step": 665 + }, + { + "KL/chosen_KL_mean": -684.744140625, + "KL/mean": -830.219482421875, + "KL/rejected_KL_mean": -975.6949462890625, + "KL/std": 363.33282470703125, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0016065204981714487, + "fcm_dpo/delta": -0.06773875653743744, + "fcm_dpo/margin": 290.9507751464844, + "fcm_dpo/q_t": 0.3932494819164276, + "grad_norm": 37.33126449584961, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": 0.6177271008491516, + "logits/rejected": 0.653031587600708, + "logps/chosen": -749.1640625, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1071.586669921875, + "loss": 1.0479, + "margin_dpo/margin_mean": 290.9507751464844, + "margin_dpo/margin_std": 353.019287109375, + "step": 666 + }, + { + "KL/chosen_KL_mean": -752.7801513671875, + "KL/mean": -917.2275390625, + "KL/rejected_KL_mean": -1081.675048828125, + "KL/std": 384.06536865234375, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0016032771673053503, + "fcm_dpo/delta": -0.12796761095523834, + "fcm_dpo/margin": 328.8948974609375, + "fcm_dpo/q_t": 0.38113367557525635, + "grad_norm": 46.49330520629883, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": 0.6967591047286987, + "logits/rejected": 0.7598137855529785, + "logps/chosen": -822.05712890625, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1169.510498046875, + "loss": 1.0041, + "margin_dpo/margin_mean": 328.8948974609375, + "margin_dpo/margin_std": 365.38958740234375, + "step": 667 + }, + { + "KL/chosen_KL_mean": -758.3023071289062, + "KL/mean": -874.9960327148438, + "KL/rejected_KL_mean": -991.6897583007812, + "KL/std": 397.096435546875, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0016032748389989138, + "fcm_dpo/delta": 0.025862596929073334, + "fcm_dpo/margin": 233.38742065429688, + "fcm_dpo/q_t": 0.415823370218277, + "grad_norm": 90.86317443847656, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": 0.693195104598999, + "logits/rejected": 0.7051081657409668, + "logps/chosen": -830.9063110351562, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1095.4288330078125, + "loss": 1.1391, + "margin_dpo/margin_mean": 233.387451171875, + "margin_dpo/margin_std": 391.69708251953125, + "step": 668 + }, + { + "KL/chosen_KL_mean": -646.857421875, + "KL/mean": -817.19384765625, + "KL/rejected_KL_mean": -987.5302734375, + "KL/std": 405.25048828125, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0015992240514606237, + "fcm_dpo/delta": -0.14561131596565247, + "fcm_dpo/margin": 340.67291259765625, + "fcm_dpo/q_t": 0.37677478790283203, + "grad_norm": 30.40396499633789, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": 0.7726386785507202, + "logits/rejected": 0.8078670501708984, + "logps/chosen": -692.9738159179688, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1065.45458984375, + "loss": 1.0074, + "margin_dpo/margin_mean": 340.67291259765625, + "margin_dpo/margin_std": 402.0104064941406, + "step": 669 + }, + { + "KL/chosen_KL_mean": -623.4371948242188, + "KL/mean": -804.9701538085938, + "KL/rejected_KL_mean": -986.5030517578125, + "KL/std": 347.4085998535156, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0015944442711770535, + "fcm_dpo/delta": -0.17989006638526917, + "fcm_dpo/margin": 363.0657958984375, + "fcm_dpo/q_t": 0.37333351373672485, + "grad_norm": 34.68000030517578, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": 0.7076517939567566, + "logits/rejected": 0.7265514135360718, + "logps/chosen": -685.782958984375, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1083.443603515625, + "loss": 0.9906, + "margin_dpo/margin_mean": 363.0657958984375, + "margin_dpo/margin_std": 421.2840270996094, + "step": 670 + }, + { + "KL/chosen_KL_mean": -731.2550048828125, + "KL/mean": -891.503173828125, + "KL/rejected_KL_mean": -1051.75146484375, + "KL/std": 348.3045959472656, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.001590254483744502, + "fcm_dpo/delta": -0.11021782457828522, + "fcm_dpo/margin": 320.496337890625, + "fcm_dpo/q_t": 0.38403597474098206, + "grad_norm": 35.5849723815918, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": 0.8641825914382935, + "logits/rejected": 0.9019452333450317, + "logps/chosen": -779.255126953125, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1135.57080078125, + "loss": 1.0174, + "margin_dpo/margin_mean": 320.496337890625, + "margin_dpo/margin_std": 362.8017272949219, + "step": 671 + }, + { + "KL/chosen_KL_mean": -793.2496337890625, + "KL/mean": -933.4834594726562, + "KL/rejected_KL_mean": -1073.71728515625, + "KL/std": 419.9259033203125, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0015874074306339025, + "fcm_dpo/delta": -0.04544100910425186, + "fcm_dpo/margin": 280.46759033203125, + "fcm_dpo/q_t": 0.40169036388397217, + "grad_norm": 47.97336196899414, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": 0.8627023696899414, + "logits/rejected": 0.894331693649292, + "logps/chosen": -851.8329467773438, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1166.857421875, + "loss": 1.0955, + "margin_dpo/margin_mean": 280.46759033203125, + "margin_dpo/margin_std": 425.1080322265625, + "step": 672 + }, + { + "KL/chosen_KL_mean": -700.2203369140625, + "KL/mean": -848.1244506835938, + "KL/rejected_KL_mean": -996.0286865234375, + "KL/std": 325.0884704589844, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.001585936057381332, + "fcm_dpo/delta": -0.06945396959781647, + "fcm_dpo/margin": 295.8083190917969, + "fcm_dpo/q_t": 0.3912809491157532, + "grad_norm": 34.701805114746094, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": 0.8700417876243591, + "logits/rejected": 0.8842042684555054, + "logps/chosen": -746.9434814453125, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1081.324951171875, + "loss": 1.0343, + "margin_dpo/margin_mean": 295.8083190917969, + "margin_dpo/margin_std": 329.3403625488281, + "step": 673 + }, + { + "KL/chosen_KL_mean": -652.2431640625, + "KL/mean": -825.498779296875, + "KL/rejected_KL_mean": -998.7545166015625, + "KL/std": 374.48016357421875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0015821305569261312, + "fcm_dpo/delta": -0.1490173041820526, + "fcm_dpo/margin": 346.5113525390625, + "fcm_dpo/q_t": 0.38002896308898926, + "grad_norm": 29.187240600585938, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": 0.822380006313324, + "logits/rejected": 0.8550535440444946, + "logps/chosen": -697.6886596679688, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1068.8004150390625, + "loss": 1.0004, + "margin_dpo/margin_mean": 346.5113525390625, + "margin_dpo/margin_std": 407.6961669921875, + "step": 674 + }, + { + "KL/chosen_KL_mean": -672.388427734375, + "KL/mean": -873.0430908203125, + "KL/rejected_KL_mean": -1073.69775390625, + "KL/std": 408.8966369628906, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0015743083786219358, + "fcm_dpo/delta": -0.2333965003490448, + "fcm_dpo/margin": 401.3092346191406, + "fcm_dpo/q_t": 0.36221542954444885, + "grad_norm": 28.302249908447266, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": 0.8097084760665894, + "logits/rejected": 0.8456708192825317, + "logps/chosen": -716.564697265625, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1147.7896728515625, + "loss": 0.9408, + "margin_dpo/margin_mean": 401.3092346191406, + "margin_dpo/margin_std": 408.553955078125, + "step": 675 + }, + { + "KL/chosen_KL_mean": -709.1005859375, + "KL/mean": -866.5613403320312, + "KL/rejected_KL_mean": -1024.0220947265625, + "KL/std": 357.8074951171875, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0015708766877651215, + "fcm_dpo/delta": -0.09517545253038406, + "fcm_dpo/margin": 314.92156982421875, + "fcm_dpo/q_t": 0.3862054646015167, + "grad_norm": 30.970930099487305, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": 0.6075150966644287, + "logits/rejected": 0.6473897099494934, + "logps/chosen": -780.4990844726562, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1112.380859375, + "loss": 1.0085, + "margin_dpo/margin_mean": 314.92156982421875, + "margin_dpo/margin_std": 317.07635498046875, + "step": 676 + }, + { + "KL/chosen_KL_mean": -727.688232421875, + "KL/mean": -877.115966796875, + "KL/rejected_KL_mean": -1026.543701171875, + "KL/std": 304.4088134765625, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0015687142731621861, + "fcm_dpo/delta": -0.06913770735263824, + "fcm_dpo/margin": 298.85552978515625, + "fcm_dpo/q_t": 0.3919876217842102, + "grad_norm": 35.21232604980469, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": 0.8967536687850952, + "logits/rejected": 0.9327912330627441, + "logps/chosen": -784.2156372070312, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1104.770263671875, + "loss": 1.0301, + "margin_dpo/margin_mean": 298.85552978515625, + "margin_dpo/margin_std": 319.7119140625, + "step": 677 + }, + { + "KL/chosen_KL_mean": -657.1640625, + "KL/mean": -838.7535400390625, + "KL/rejected_KL_mean": -1020.3428955078125, + "KL/std": 390.63763427734375, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0015646331012248993, + "fcm_dpo/delta": -0.16916753351688385, + "fcm_dpo/margin": 363.1788024902344, + "fcm_dpo/q_t": 0.37448346614837646, + "grad_norm": 34.81563186645508, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": 0.7971528768539429, + "logits/rejected": 0.839873731136322, + "logps/chosen": -703.298583984375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1100.947509765625, + "loss": 0.9893, + "margin_dpo/margin_mean": 363.1788330078125, + "margin_dpo/margin_std": 414.970947265625, + "step": 678 + }, + { + "KL/chosen_KL_mean": -696.775146484375, + "KL/mean": -835.5936279296875, + "KL/rejected_KL_mean": -974.4121704101562, + "KL/std": 363.2303466796875, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.00156167964451015, + "fcm_dpo/delta": -0.03373512253165245, + "fcm_dpo/margin": 277.6371154785156, + "fcm_dpo/q_t": 0.4013155698776245, + "grad_norm": 38.71091079711914, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": 0.7600429058074951, + "logits/rejected": 0.7770619988441467, + "logps/chosen": -747.070068359375, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1051.01025390625, + "loss": 1.0766, + "margin_dpo/margin_mean": 277.6371154785156, + "margin_dpo/margin_std": 370.28955078125, + "step": 679 + }, + { + "KL/chosen_KL_mean": -711.7872314453125, + "KL/mean": -888.6114501953125, + "KL/rejected_KL_mean": -1065.435546875, + "KL/std": 424.00201416015625, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0015581029001623392, + "fcm_dpo/delta": -0.15185554325580597, + "fcm_dpo/margin": 353.6483459472656, + "fcm_dpo/q_t": 0.37735500931739807, + "grad_norm": 30.313135147094727, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": 0.6791744828224182, + "logits/rejected": 0.7271414995193481, + "logps/chosen": -788.7030029296875, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1177.8203125, + "loss": 1.0183, + "margin_dpo/margin_mean": 353.64837646484375, + "margin_dpo/margin_std": 448.4976806640625, + "step": 680 + }, + { + "KL/chosen_KL_mean": -720.9833374023438, + "KL/mean": -870.0701904296875, + "KL/rejected_KL_mean": -1019.156982421875, + "KL/std": 374.87548828125, + "epoch": 1.0, + "fcm_dpo/beta": 0.001556276110932231, + "fcm_dpo/delta": -0.06436647474765778, + "fcm_dpo/margin": 298.173583984375, + "fcm_dpo/q_t": 0.394231379032135, + "grad_norm": 32.9175910949707, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": 0.7696690559387207, + "logits/rejected": 0.7999899387359619, + "logps/chosen": -781.9406127929688, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1107.7149658203125, + "loss": 1.0477, + "margin_dpo/margin_mean": 298.173583984375, + "margin_dpo/margin_std": 348.46051025390625, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 0.9659692127799148, + "train_runtime": 1735.4678, + "train_samples_per_second": 25.122, + "train_steps_per_second": 0.392 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}