From 0122555ec684c9712f07cfb377d405b58be4fa40 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 1 Jun 2026 07:43:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15246 +++++++++++++++++++++++++++++ 18 files changed, 17809 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..33e6682 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-8 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..06af4a6 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.370076598991244, + "train_runtime": 1650.2951, + "train_samples": 42336, + "train_samples_per_second": 25.654, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..273e3df --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57b1682ff8c9b92bf4c25962fbf2ba80147aec9cf07ad085c0cca4cce6edd21 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..73458d7 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caba6d9bb7c4fcc9be34923ed0c5acbd5fe962bc6c4dea8d163d24da8fc85947 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..db9bb9a --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bf4f3a78492a72292c8db3067c883c24ba19780d93d3d6820e82fd0e963c6d7 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..6b24e10 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf507d5198e4c40c38ea21ada333fa38ecd150df09fc3b729cead22cb89aabd +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..30b640b --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5184d70d8f88ae8efe03c950fa3347b1a34e72049072c329a3ae195157a8569c +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..cb23ac0 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83acd57a1f32924009c38af25152aa3c1c68934f2bf58614d3ba95d72fb61952 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..aa600a7 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66bcab9fb3e44627ce989ac90fce68eac68834d2aa87daf7e3a0bb8abb894573 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..06af4a6 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.370076598991244, + "train_runtime": 1650.2951, + "train_samples": 42336, + "train_samples_per_second": 25.654, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..061ee42 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000336766242981, + "grad_norm": 28.2196044921875, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.4990639090538025, + "grad_norm": 27.82891273498535, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.0621337890625, + "KL/mean": 0.06413677334785461, + "KL/rejected_KL_mean": 0.06613540649414062, + "KL/std": 0.22972196340560913, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0040013790130615234, + "fcm_dpo/q_t": 0.5000998377799988, + "grad_norm": 31.116546630859375, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09941669553518295, + "logits/rejected": 0.06143157184123993, + "logps/chosen": -65.39512634277344, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.76239776611328, + "loss": 1.3869, + "margin_dpo/margin_mean": -0.00400128960609436, + "margin_dpo/margin_std": 0.290522038936615, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.000263214111328125, + "KL/mean": -0.023683160543441772, + "KL/rejected_KL_mean": -0.047637939453125, + "KL/std": 0.24327343702316284, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04789847135543823, + "fcm_dpo/q_t": 0.49880456924438477, + "grad_norm": 34.16933059692383, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.11254644393920898, + "logits/rejected": 0.0962742418050766, + "logps/chosen": -76.85992431640625, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.96287536621094, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.04789838194847107, + "margin_dpo/margin_std": 0.3381476402282715, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.027805328369140625, + "KL/mean": -0.015260711312294006, + "KL/rejected_KL_mean": -0.002712249755859375, + "KL/std": 0.18707606196403503, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.025092214345932007, + "fcm_dpo/q_t": 0.5006270408630371, + "grad_norm": 29.659147262573242, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.07833173871040344, + "logits/rejected": 0.03964802622795105, + "logps/chosen": -62.9991455078125, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.92192077636719, + "loss": 1.3889, + "margin_dpo/margin_mean": -0.02509164810180664, + "margin_dpo/margin_std": 0.23738428950309753, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.0323944091796875, + "KL/mean": 0.007522225379943848, + "KL/rejected_KL_mean": 0.047443389892578125, + "KL/std": 0.25012582540512085, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.07983356714248657, + "fcm_dpo/q_t": 0.501994788646698, + "grad_norm": 29.858251571655273, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.14125987887382507, + "logits/rejected": 0.10181364417076111, + "logps/chosen": -51.339759826660156, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.7249526977539, + "loss": 1.3945, + "margin_dpo/margin_mean": -0.07983347773551941, + "margin_dpo/margin_std": 0.3165475130081177, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.05200386047363281, + "KL/mean": 0.038868606090545654, + "KL/rejected_KL_mean": 0.025732040405273438, + "KL/std": 0.21536785364151, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02626955509185791, + "fcm_dpo/q_t": 0.4993433356285095, + "grad_norm": 27.112102508544922, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.019042763859033585, + "logits/rejected": -0.02486884593963623, + "logps/chosen": -51.40740966796875, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.3570785522461, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.026269584894180298, + "margin_dpo/margin_std": 0.3134131133556366, + "step": 7 + }, + { + "KL/chosen_KL_mean": -0.024341583251953125, + "KL/mean": -0.011224046349525452, + "KL/rejected_KL_mean": 0.001895904541015625, + "KL/std": 0.2582741677761078, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.026233285665512085, + "fcm_dpo/q_t": 0.5006557703018188, + "grad_norm": 28.65567970275879, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.07061031460762024, + "logits/rejected": 0.04848004877567291, + "logps/chosen": -62.221885681152344, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.659912109375, + "loss": 1.3892, + "margin_dpo/margin_mean": -0.026233792304992676, + "margin_dpo/margin_std": 0.357799768447876, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.012987136840820312, + "KL/mean": -0.02591407299041748, + "KL/rejected_KL_mean": -0.038845062255859375, + "KL/std": 0.2304016649723053, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.025857150554656982, + "fcm_dpo/q_t": 0.4993537664413452, + "grad_norm": 31.47567367553711, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.15661674737930298, + "logits/rejected": 0.09843897819519043, + "logps/chosen": -55.64270782470703, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.25106811523438, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.02585729956626892, + "margin_dpo/margin_std": 0.3030700385570526, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.0495147705078125, + "KL/mean": -0.027920737862586975, + "KL/rejected_KL_mean": -0.006328582763671875, + "KL/std": 0.25644826889038086, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.043188005685806274, + "fcm_dpo/q_t": 0.5010795593261719, + "grad_norm": 29.739234924316406, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.12492163479328156, + "logits/rejected": 0.09430065751075745, + "logps/chosen": -62.740116119384766, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.61643981933594, + "loss": 1.391, + "margin_dpo/margin_mean": -0.043188244104385376, + "margin_dpo/margin_std": 0.3699379861354828, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.019266128540039062, + "KL/mean": -0.0006564557552337646, + "KL/rejected_KL_mean": -0.0205841064453125, + "KL/std": 0.20153909921646118, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03984951972961426, + "fcm_dpo/q_t": 0.49900442361831665, + "grad_norm": 29.289079666137695, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.1184384673833847, + "logits/rejected": 0.11146491020917892, + "logps/chosen": -65.74784851074219, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.4970474243164, + "loss": 1.3826, + "margin_dpo/margin_mean": 0.039849698543548584, + "margin_dpo/margin_std": 0.301962286233902, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.03316497802734375, + "KL/mean": -0.036665767431259155, + "KL/rejected_KL_mean": -0.04016876220703125, + "KL/std": 0.2160276174545288, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.007006347179412842, + "fcm_dpo/q_t": 0.4998248219490051, + "grad_norm": 28.296424865722656, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.00660199299454689, + "logits/rejected": -0.009146112017333508, + "logps/chosen": -60.73805618286133, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.455810546875, + "loss": 1.3858, + "margin_dpo/margin_mean": 0.007006734609603882, + "margin_dpo/margin_std": 0.2850860357284546, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.041828155517578125, + "KL/mean": 0.04208873212337494, + "KL/rejected_KL_mean": 0.0423583984375, + "KL/std": 0.19688871502876282, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.000529557466506958, + "fcm_dpo/q_t": 0.5000130534172058, + "grad_norm": 29.095415115356445, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.10334185510873795, + "logits/rejected": 0.04153688997030258, + "logps/chosen": -49.867427825927734, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.33582305908203, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0005303919315338135, + "margin_dpo/margin_std": 0.302842915058136, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.022100448608398438, + "KL/mean": 0.00214308500289917, + "KL/rejected_KL_mean": -0.017810821533203125, + "KL/std": 0.17682762444019318, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03991013765335083, + "fcm_dpo/q_t": 0.4990024268627167, + "grad_norm": 29.589576721191406, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.0741623118519783, + "logits/rejected": 0.056773897260427475, + "logps/chosen": -60.59669494628906, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.81086730957031, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.039910584688186646, + "margin_dpo/margin_std": 0.25130391120910645, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.03235435485839844, + "KL/mean": -0.033513978123664856, + "KL/rejected_KL_mean": -0.03466796875, + "KL/std": 0.23530861735343933, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0023103058338165283, + "fcm_dpo/q_t": 0.4999410808086395, + "grad_norm": 33.306461334228516, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.07059284299612045, + "logits/rejected": 0.027335654944181442, + "logps/chosen": -63.50189208984375, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.92417907714844, + "loss": 1.3864, + "margin_dpo/margin_mean": 0.0023106038570404053, + "margin_dpo/margin_std": 0.35110729932785034, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.0051555633544921875, + "KL/mean": -0.00991896539926529, + "KL/rejected_KL_mean": -0.024997711181640625, + "KL/std": 0.20466622710227966, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03015093505382538, + "fcm_dpo/q_t": 0.49924665689468384, + "grad_norm": 26.78266716003418, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.11953230202198029, + "logits/rejected": 0.08214319497346878, + "logps/chosen": -46.52714538574219, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.30033874511719, + "loss": 1.3835, + "margin_dpo/margin_mean": 0.030150994658470154, + "margin_dpo/margin_std": 0.3233643174171448, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.03330421447753906, + "KL/mean": -0.026638299226760864, + "KL/rejected_KL_mean": -0.0199737548828125, + "KL/std": 0.21004769206047058, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.013328969478607178, + "fcm_dpo/q_t": 0.5003334283828735, + "grad_norm": 32.7893180847168, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.05357828736305237, + "logits/rejected": 0.03485024347901344, + "logps/chosen": -64.11113739013672, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.42874145507812, + "loss": 1.3878, + "margin_dpo/margin_mean": -0.013329803943634033, + "margin_dpo/margin_std": 0.27140507102012634, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.001186370849609375, + "KL/mean": -0.005001351237297058, + "KL/rejected_KL_mean": -0.008813858032226562, + "KL/std": 0.21888390183448792, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.007630079984664917, + "fcm_dpo/q_t": 0.4998093545436859, + "grad_norm": 27.819944381713867, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.0680946335196495, + "logits/rejected": 0.024525661021471024, + "logps/chosen": -44.87552261352539, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -70.98486328125, + "loss": 1.3858, + "margin_dpo/margin_mean": 0.007630020380020142, + "margin_dpo/margin_std": 0.3008061647415161, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.01468658447265625, + "KL/mean": -0.019955851137638092, + "KL/rejected_KL_mean": -0.054599761962890625, + "KL/std": 0.20606964826583862, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06928969919681549, + "fcm_dpo/q_t": 0.4982680082321167, + "grad_norm": 30.73091697692871, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.0740823969244957, + "logits/rejected": 0.06053391098976135, + "logps/chosen": -68.14512634277344, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.22599029541016, + "loss": 1.3795, + "margin_dpo/margin_mean": 0.06928981840610504, + "margin_dpo/margin_std": 0.22947733104228973, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.0164337158203125, + "KL/mean": 0.01836688071489334, + "KL/rejected_KL_mean": 0.020305633544921875, + "KL/std": 0.2009357511997223, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.003872022032737732, + "fcm_dpo/q_t": 0.5000965595245361, + "grad_norm": 29.24262046813965, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.14226043224334717, + "logits/rejected": 0.1189441829919815, + "logps/chosen": -53.66212844848633, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.14881134033203, + "loss": 1.3869, + "margin_dpo/margin_mean": -0.0038716942071914673, + "margin_dpo/margin_std": 0.2613745629787445, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.013164520263671875, + "KL/mean": -0.004536345601081848, + "KL/rejected_KL_mean": 0.00409698486328125, + "KL/std": 0.20376834273338318, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.017256349325180054, + "fcm_dpo/q_t": 0.5004312992095947, + "grad_norm": 29.48101806640625, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.1120043694972992, + "logits/rejected": 0.08625894784927368, + "logps/chosen": -64.7147216796875, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -81.016845703125, + "loss": 1.3882, + "margin_dpo/margin_mean": -0.01725691556930542, + "margin_dpo/margin_std": 0.2752753496170044, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.0014190673828125, + "KL/mean": -0.024805426597595215, + "KL/rejected_KL_mean": -0.051029205322265625, + "KL/std": 0.2457861751317978, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05245363712310791, + "fcm_dpo/q_t": 0.49868905544281006, + "grad_norm": 28.892513275146484, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": -0.0012458395212888718, + "logits/rejected": -0.02185724303126335, + "logps/chosen": -58.03457260131836, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.77824401855469, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.052453696727752686, + "margin_dpo/margin_std": 0.36326271295547485, + "step": 22 + }, + { + "KL/chosen_KL_mean": -0.005031585693359375, + "KL/mean": -0.004315197467803955, + "KL/rejected_KL_mean": -0.003604888916015625, + "KL/std": 0.22622814774513245, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0014243721961975098, + "fcm_dpo/q_t": 0.5000349879264832, + "grad_norm": 32.9621696472168, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.14867977797985077, + "logits/rejected": 0.12283647060394287, + "logps/chosen": -66.36111450195312, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.03129577636719, + "loss": 1.3867, + "margin_dpo/margin_mean": -0.0014249980449676514, + "margin_dpo/margin_std": 0.3283410668373108, + "step": 23 + }, + { + "KL/chosen_KL_mean": -0.023939132690429688, + "KL/mean": -0.010058224201202393, + "KL/rejected_KL_mean": 0.0038280487060546875, + "KL/std": 0.2233589142560959, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.027758657932281494, + "fcm_dpo/q_t": 0.5006934404373169, + "grad_norm": 26.37616539001465, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.11849376559257507, + "logits/rejected": 0.08617529273033142, + "logps/chosen": -54.48517608642578, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.3343505859375, + "loss": 1.3893, + "margin_dpo/margin_mean": -0.027759432792663574, + "margin_dpo/margin_std": 0.3043414354324341, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.07821273803710938, + "KL/mean": -0.042458370327949524, + "KL/rejected_KL_mean": -0.00670623779296875, + "KL/std": 0.2250480055809021, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0715077817440033, + "fcm_dpo/q_t": 0.5017871856689453, + "grad_norm": 29.881423950195312, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.10595692694187164, + "logits/rejected": 0.05439333990216255, + "logps/chosen": -60.08241653442383, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.48046875, + "loss": 1.3937, + "margin_dpo/margin_mean": -0.07150781154632568, + "margin_dpo/margin_std": 0.2951575517654419, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.025112152099609375, + "KL/mean": -0.030633173882961273, + "KL/rejected_KL_mean": -0.036151885986328125, + "KL/std": 0.2159086912870407, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011040762066841125, + "fcm_dpo/q_t": 0.4997239112854004, + "grad_norm": 29.78719711303711, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.10197645425796509, + "logits/rejected": 0.08369824290275574, + "logps/chosen": -56.844268798828125, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.87948608398438, + "loss": 1.3855, + "margin_dpo/margin_mean": 0.011040925979614258, + "margin_dpo/margin_std": 0.3176705837249756, + "step": 26 + }, + { + "KL/chosen_KL_mean": -0.040157318115234375, + "KL/mean": -0.01530282199382782, + "KL/rejected_KL_mean": 0.009548187255859375, + "KL/std": 0.24493414163589478, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04971262812614441, + "fcm_dpo/q_t": 0.5012425184249878, + "grad_norm": 29.07723617553711, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.0994294285774231, + "logits/rejected": 0.07478873431682587, + "logps/chosen": -62.91718292236328, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.33482360839844, + "loss": 1.3915, + "margin_dpo/margin_mean": -0.04971283674240112, + "margin_dpo/margin_std": 0.2961461544036865, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.025875091552734375, + "KL/mean": -0.01864849030971527, + "KL/rejected_KL_mean": -0.011423110961914062, + "KL/std": 0.2145068347454071, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.014449745416641235, + "fcm_dpo/q_t": 0.500361442565918, + "grad_norm": 27.64870834350586, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.061025142669677734, + "logits/rejected": 0.05223534256219864, + "logps/chosen": -59.8592529296875, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.40946197509766, + "loss": 1.388, + "margin_dpo/margin_mean": -0.014449775218963623, + "margin_dpo/margin_std": 0.28998634219169617, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.011623382568359375, + "KL/mean": -0.024745166301727295, + "KL/rejected_KL_mean": -0.03786468505859375, + "KL/std": 0.22506870329380035, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02624613046646118, + "fcm_dpo/q_t": 0.49934351444244385, + "grad_norm": 32.635955810546875, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.1302889883518219, + "logits/rejected": 0.11286543309688568, + "logps/chosen": -74.13182067871094, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.36885070800781, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.026246458292007446, + "margin_dpo/margin_std": 0.3253002464771271, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.0457916259765625, + "KL/mean": -0.03270387649536133, + "KL/rejected_KL_mean": -0.01961517333984375, + "KL/std": 0.2302623689174652, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.026176929473876953, + "fcm_dpo/q_t": 0.5006539225578308, + "grad_norm": 30.555559158325195, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.12713855504989624, + "logits/rejected": 0.07250553369522095, + "logps/chosen": -50.797080993652344, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.31024932861328, + "loss": 1.3891, + "margin_dpo/margin_mean": -0.026176661252975464, + "margin_dpo/margin_std": 0.2921898066997528, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.028470993041992188, + "KL/mean": -0.032488495111465454, + "KL/rejected_KL_mean": -0.03650665283203125, + "KL/std": 0.22643673419952393, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.00803285837173462, + "fcm_dpo/q_t": 0.49979931116104126, + "grad_norm": 34.28264617919922, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.09353435784578323, + "logits/rejected": 0.04833420366048813, + "logps/chosen": -65.36522674560547, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.80316925048828, + "loss": 1.3857, + "margin_dpo/margin_mean": 0.008032828569412231, + "margin_dpo/margin_std": 0.31350964307785034, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.023860931396484375, + "KL/mean": -0.0067330896854400635, + "KL/rejected_KL_mean": -0.037326812744140625, + "KL/std": 0.2645628750324249, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.061192214488983154, + "fcm_dpo/q_t": 0.49847108125686646, + "grad_norm": 29.951520919799805, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.06752718985080719, + "logits/rejected": 0.059862636029720306, + "logps/chosen": -67.15946960449219, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.84496307373047, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06119275093078613, + "margin_dpo/margin_std": 0.3593711256980896, + "step": 32 + }, + { + "KL/chosen_KL_mean": -0.0693359375, + "KL/mean": -0.0568530336022377, + "KL/rejected_KL_mean": -0.04436492919921875, + "KL/std": 0.23845943808555603, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0249737948179245, + "fcm_dpo/q_t": 0.5006238222122192, + "grad_norm": 31.129165649414062, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.026731964200735092, + "logits/rejected": 0.0009631272405385971, + "logps/chosen": -64.10881805419922, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.72795104980469, + "loss": 1.3891, + "margin_dpo/margin_mean": -0.024973228573799133, + "margin_dpo/margin_std": 0.34394294023513794, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.00998687744140625, + "KL/mean": -0.04686702787876129, + "KL/rejected_KL_mean": -0.08374404907226562, + "KL/std": 0.2107549011707306, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07376006245613098, + "fcm_dpo/q_t": 0.4981565773487091, + "grad_norm": 28.369434356689453, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.11266010254621506, + "logits/rejected": 0.08217991888523102, + "logps/chosen": -53.67428207397461, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.8636474609375, + "loss": 1.3792, + "margin_dpo/margin_mean": 0.07376009225845337, + "margin_dpo/margin_std": 0.28919702768325806, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.05603218078613281, + "KL/mean": -0.08405140042304993, + "KL/rejected_KL_mean": -0.11207199096679688, + "KL/std": 0.21035067737102509, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.056039512157440186, + "fcm_dpo/q_t": 0.4985993802547455, + "grad_norm": 27.643247604370117, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.04645001143217087, + "logits/rejected": 0.024315200746059418, + "logps/chosen": -61.07289505004883, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.89805603027344, + "loss": 1.3809, + "margin_dpo/margin_mean": 0.05603942275047302, + "margin_dpo/margin_std": 0.2715159058570862, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.06983184814453125, + "KL/mean": -0.08555865287780762, + "KL/rejected_KL_mean": -0.10128402709960938, + "KL/std": 0.23037990927696228, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03145325183868408, + "fcm_dpo/q_t": 0.49921417236328125, + "grad_norm": 28.695158004760742, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.09576475620269775, + "logits/rejected": 0.043359238654375076, + "logps/chosen": -50.60719299316406, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.21806335449219, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.03145378828048706, + "margin_dpo/margin_std": 0.34016674757003784, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.032245635986328125, + "KL/mean": -0.10205593705177307, + "KL/rejected_KL_mean": -0.17187118530273438, + "KL/std": 0.2938510775566101, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13962048292160034, + "fcm_dpo/q_t": 0.4965134263038635, + "grad_norm": 37.48423767089844, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.10579690337181091, + "logits/rejected": 0.025799859315156937, + "logps/chosen": -59.5861930847168, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.44889831542969, + "loss": 1.3729, + "margin_dpo/margin_mean": 0.13962051272392273, + "margin_dpo/margin_std": 0.4410157799720764, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.0857086181640625, + "KL/mean": -0.11043363809585571, + "KL/rejected_KL_mean": -0.13515663146972656, + "KL/std": 0.25614839792251587, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04944509267807007, + "fcm_dpo/q_t": 0.4987648129463196, + "grad_norm": 29.67615509033203, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.08986853808164597, + "logits/rejected": 0.0756976380944252, + "logps/chosen": -65.87406921386719, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.2971420288086, + "loss": 1.3817, + "margin_dpo/margin_mean": 0.04944491386413574, + "margin_dpo/margin_std": 0.3537583351135254, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.10344505310058594, + "KL/mean": -0.12965010106563568, + "KL/rejected_KL_mean": -0.155853271484375, + "KL/std": 0.25784462690353394, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05241057276725769, + "fcm_dpo/q_t": 0.4986896514892578, + "grad_norm": 29.232746124267578, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.1692015826702118, + "logits/rejected": 0.14214356243610382, + "logps/chosen": -57.28025436401367, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.64218139648438, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.052410632371902466, + "margin_dpo/margin_std": 0.3558116555213928, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.11075210571289062, + "KL/mean": -0.1190851703286171, + "KL/rejected_KL_mean": -0.12742233276367188, + "KL/std": 0.24136286973953247, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.016666367650032043, + "fcm_dpo/q_t": 0.49958330392837524, + "grad_norm": 31.398401260375977, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.1105370819568634, + "logits/rejected": 0.06097438558936119, + "logps/chosen": -61.44491958618164, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.23439025878906, + "loss": 1.3849, + "margin_dpo/margin_mean": 0.016666486859321594, + "margin_dpo/margin_std": 0.327433705329895, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.11747360229492188, + "KL/mean": -0.16613994538784027, + "KL/rejected_KL_mean": -0.21480178833007812, + "KL/std": 0.2806848883628845, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09732410311698914, + "fcm_dpo/q_t": 0.49756765365600586, + "grad_norm": 30.464834213256836, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.06427644938230515, + "logits/rejected": 0.04391469433903694, + "logps/chosen": -67.66419982910156, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.09268951416016, + "loss": 1.377, + "margin_dpo/margin_mean": 0.09732472896575928, + "margin_dpo/margin_std": 0.3751354217529297, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.10732078552246094, + "KL/mean": -0.1636602282524109, + "KL/rejected_KL_mean": -0.22000503540039062, + "KL/std": 0.3067892789840698, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11268658936023712, + "fcm_dpo/q_t": 0.4971844553947449, + "grad_norm": 29.092308044433594, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.05194007605314255, + "logits/rejected": 0.0304647795855999, + "logps/chosen": -61.37217712402344, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.58291625976562, + "loss": 1.3755, + "margin_dpo/margin_mean": 0.1126861423254013, + "margin_dpo/margin_std": 0.4176199436187744, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.09345245361328125, + "KL/mean": -0.111243337392807, + "KL/rejected_KL_mean": -0.12903594970703125, + "KL/std": 0.28760355710983276, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.035582125186920166, + "fcm_dpo/q_t": 0.49911123514175415, + "grad_norm": 34.529136657714844, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.08271262049674988, + "logits/rejected": 0.07184702157974243, + "logps/chosen": -71.90248107910156, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.25367736816406, + "loss": 1.3832, + "margin_dpo/margin_mean": 0.035581886768341064, + "margin_dpo/margin_std": 0.3950834274291992, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.16973114013671875, + "KL/mean": -0.1733190417289734, + "KL/rejected_KL_mean": -0.176910400390625, + "KL/std": 0.3061409592628479, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.007180750370025635, + "fcm_dpo/q_t": 0.4998205602169037, + "grad_norm": 32.519447326660156, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.04154222086071968, + "logits/rejected": 0.011357433162629604, + "logps/chosen": -66.72016143798828, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.2388916015625, + "loss": 1.3861, + "margin_dpo/margin_mean": 0.0071804821491241455, + "margin_dpo/margin_std": 0.45991164445877075, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.13983726501464844, + "KL/mean": -0.17917409539222717, + "KL/rejected_KL_mean": -0.21851348876953125, + "KL/std": 0.2951315641403198, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0786718875169754, + "fcm_dpo/q_t": 0.49803340435028076, + "grad_norm": 31.8658504486084, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.13182783126831055, + "logits/rejected": 0.07822854816913605, + "logps/chosen": -62.383689880371094, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.1851806640625, + "loss": 1.3789, + "margin_dpo/margin_mean": 0.07867246866226196, + "margin_dpo/margin_std": 0.4071173369884491, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.11602401733398438, + "KL/mean": -0.19016796350479126, + "KL/rejected_KL_mean": -0.2643165588378906, + "KL/std": 0.31932127475738525, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.14828699827194214, + "fcm_dpo/q_t": 0.4962952733039856, + "grad_norm": 31.104398727416992, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.13636288046836853, + "logits/rejected": 0.08986735343933105, + "logps/chosen": -61.61492919921875, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.17604064941406, + "loss": 1.372, + "margin_dpo/margin_mean": 0.14828673005104065, + "margin_dpo/margin_std": 0.4339235723018646, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.14241790771484375, + "KL/mean": -0.20898690819740295, + "KL/rejected_KL_mean": -0.2755584716796875, + "KL/std": 0.2778022289276123, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13313710689544678, + "fcm_dpo/q_t": 0.49667197465896606, + "grad_norm": 28.65785789489746, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.05303880572319031, + "logits/rejected": 0.009075134992599487, + "logps/chosen": -51.72076416015625, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.49711608886719, + "loss": 1.3734, + "margin_dpo/margin_mean": 0.13313719630241394, + "margin_dpo/margin_std": 0.38025137782096863, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.2553062438964844, + "KL/mean": -0.2546156048774719, + "KL/rejected_KL_mean": -0.2539253234863281, + "KL/std": 0.34144163131713867, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013832151889801025, + "fcm_dpo/q_t": 0.5000360608100891, + "grad_norm": 26.89451789855957, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.16787520051002502, + "logits/rejected": 0.1375715285539627, + "logps/chosen": -52.04895782470703, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.47896575927734, + "loss": 1.3871, + "margin_dpo/margin_mean": -0.0013829171657562256, + "margin_dpo/margin_std": 0.4911916255950928, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.20473098754882812, + "KL/mean": -0.23833119869232178, + "KL/rejected_KL_mean": -0.2719306945800781, + "KL/std": 0.333074688911438, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06719930469989777, + "fcm_dpo/q_t": 0.4983193874359131, + "grad_norm": 27.35091781616211, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.03351443260908127, + "logits/rejected": 0.011932183057069778, + "logps/chosen": -58.339332580566406, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.90399932861328, + "loss": 1.3801, + "margin_dpo/margin_mean": 0.06719934940338135, + "margin_dpo/margin_std": 0.4344923794269562, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.25493812561035156, + "KL/mean": -0.3313453197479248, + "KL/rejected_KL_mean": -0.40775299072265625, + "KL/std": 0.3598587214946747, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15281718969345093, + "fcm_dpo/q_t": 0.4961821436882019, + "grad_norm": 27.900310516357422, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.11298177391290665, + "logits/rejected": 0.08316892385482788, + "logps/chosen": -53.11137771606445, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.58235168457031, + "loss": 1.3715, + "margin_dpo/margin_mean": 0.15281730890274048, + "margin_dpo/margin_std": 0.40361297130584717, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.3104381561279297, + "KL/mean": -0.41523122787475586, + "KL/rejected_KL_mean": -0.5200233459472656, + "KL/std": 0.4115421772003174, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20958328247070312, + "fcm_dpo/q_t": 0.49476635456085205, + "grad_norm": 30.999216079711914, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.07924918085336685, + "logits/rejected": 0.05170644819736481, + "logps/chosen": -63.966880798339844, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.6523208618164, + "loss": 1.3661, + "margin_dpo/margin_mean": 0.209583580493927, + "margin_dpo/margin_std": 0.49226850271224976, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.32818603515625, + "KL/mean": -0.4305916428565979, + "KL/rejected_KL_mean": -0.5329971313476562, + "KL/std": 0.40797537565231323, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2048064023256302, + "fcm_dpo/q_t": 0.4948863089084625, + "grad_norm": 32.73704528808594, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.07293161749839783, + "logits/rejected": 0.024682261049747467, + "logps/chosen": -68.16839599609375, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.50390625, + "loss": 1.3666, + "margin_dpo/margin_mean": 0.2048058956861496, + "margin_dpo/margin_std": 0.5083526372909546, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.34824371337890625, + "KL/mean": -0.43194323778152466, + "KL/rejected_KL_mean": -0.5156440734863281, + "KL/std": 0.3921298384666443, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1674003303050995, + "fcm_dpo/q_t": 0.4958154559135437, + "grad_norm": 27.480405807495117, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.07155004888772964, + "logits/rejected": 0.06118874251842499, + "logps/chosen": -57.226383209228516, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.271339416503906, + "loss": 1.3702, + "margin_dpo/margin_mean": 0.1674005091190338, + "margin_dpo/margin_std": 0.4693652391433716, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3873310089111328, + "KL/mean": -0.46201127767562866, + "KL/rejected_KL_mean": -0.5366916656494141, + "KL/std": 0.39615002274513245, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.14936020970344543, + "fcm_dpo/q_t": 0.4962688088417053, + "grad_norm": 26.520944595336914, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.03022252582013607, + "logits/rejected": 0.015316802076995373, + "logps/chosen": -47.65425109863281, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.73095703125, + "loss": 1.3721, + "margin_dpo/margin_mean": 0.14936044812202454, + "margin_dpo/margin_std": 0.5132778286933899, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.3409919738769531, + "KL/mean": -0.4967942535877228, + "KL/rejected_KL_mean": -0.6526031494140625, + "KL/std": 0.48860490322113037, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3116055130958557, + "fcm_dpo/q_t": 0.49222302436828613, + "grad_norm": 31.283021926879883, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.04874131828546524, + "logits/rejected": -0.029031695798039436, + "logps/chosen": -50.66718292236328, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.09649658203125, + "loss": 1.3565, + "margin_dpo/margin_mean": 0.31160497665405273, + "margin_dpo/margin_std": 0.6381343603134155, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.34792327880859375, + "KL/mean": -0.44482094049453735, + "KL/rejected_KL_mean": -0.5417137145996094, + "KL/std": 0.4808979034423828, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19379210472106934, + "fcm_dpo/q_t": 0.49515196681022644, + "grad_norm": 27.554279327392578, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.13023659586906433, + "logits/rejected": 0.10778436064720154, + "logps/chosen": -57.11489486694336, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.84675598144531, + "loss": 1.3682, + "margin_dpo/margin_mean": 0.19379198551177979, + "margin_dpo/margin_std": 0.671954333782196, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.5123443603515625, + "KL/mean": -0.6745944023132324, + "KL/rejected_KL_mean": -0.8368415832519531, + "KL/std": 0.6111703515052795, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3244953751564026, + "fcm_dpo/q_t": 0.49190065264701843, + "grad_norm": 30.53541374206543, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.12122000753879547, + "logits/rejected": 0.05556311830878258, + "logps/chosen": -58.28009033203125, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.59382629394531, + "loss": 1.3553, + "margin_dpo/margin_mean": 0.3244949281215668, + "margin_dpo/margin_std": 0.6604194641113281, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.5639533996582031, + "KL/mean": -0.6952039003372192, + "KL/rejected_KL_mean": -0.8264541625976562, + "KL/std": 0.6729289889335632, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.26249879598617554, + "fcm_dpo/q_t": 0.4935028851032257, + "grad_norm": 30.190555572509766, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.05941343307495117, + "logits/rejected": 0.04385446757078171, + "logps/chosen": -73.32804107666016, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.31920623779297, + "loss": 1.363, + "margin_dpo/margin_mean": 0.2624986469745636, + "margin_dpo/margin_std": 0.9827470183372498, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.5376720428466797, + "KL/mean": -0.6911669969558716, + "KL/rejected_KL_mean": -0.8446674346923828, + "KL/std": 0.6432867050170898, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3069899380207062, + "fcm_dpo/q_t": 0.4923567771911621, + "grad_norm": 26.56601905822754, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.12795722484588623, + "logits/rejected": 0.06226852163672447, + "logps/chosen": -50.35844802856445, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -77.98834991455078, + "loss": 1.3577, + "margin_dpo/margin_mean": 0.3069910705089569, + "margin_dpo/margin_std": 0.8052564859390259, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.6147499084472656, + "KL/mean": -0.6438243389129639, + "KL/rejected_KL_mean": -0.6729011535644531, + "KL/std": 0.6342208981513977, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05814960598945618, + "fcm_dpo/q_t": 0.498546302318573, + "grad_norm": 29.609012603759766, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.13567467033863068, + "logits/rejected": 0.13422152400016785, + "logps/chosen": -63.83952331542969, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -62.03337860107422, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.05814942717552185, + "margin_dpo/margin_std": 0.809841513633728, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.7202339172363281, + "KL/mean": -0.7759703397750854, + "KL/rejected_KL_mean": -0.8317070007324219, + "KL/std": 0.7063132524490356, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11147007346153259, + "fcm_dpo/q_t": 0.49721381068229675, + "grad_norm": 27.97510528564453, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.14454835653305054, + "logits/rejected": 0.11166666448116302, + "logps/chosen": -49.737030029296875, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.73987579345703, + "loss": 1.3776, + "margin_dpo/margin_mean": 0.11146968603134155, + "margin_dpo/margin_std": 0.9815603494644165, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.7713375091552734, + "KL/mean": -0.8586764931678772, + "KL/rejected_KL_mean": -0.9460220336914062, + "KL/std": 0.6308267712593079, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1746814250946045, + "fcm_dpo/q_t": 0.4956405758857727, + "grad_norm": 28.683074951171875, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.09312943369150162, + "logits/rejected": 0.0548982173204422, + "logps/chosen": -63.52320861816406, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.87963104248047, + "loss": 1.3708, + "margin_dpo/margin_mean": 0.1746809184551239, + "margin_dpo/margin_std": 0.8726956248283386, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.5630893707275391, + "KL/mean": -0.8304413557052612, + "KL/rejected_KL_mean": -1.0978012084960938, + "KL/std": 0.6817377209663391, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5347055196762085, + "fcm_dpo/q_t": 0.48666688799858093, + "grad_norm": 31.4731502532959, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.1990186870098114, + "logits/rejected": 0.1737050712108612, + "logps/chosen": -61.078338623046875, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -86.20801544189453, + "loss": 1.3355, + "margin_dpo/margin_mean": 0.534705638885498, + "margin_dpo/margin_std": 0.8916192054748535, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.8345146179199219, + "KL/mean": -0.8636895418167114, + "KL/rejected_KL_mean": -0.8928680419921875, + "KL/std": 0.7093801498413086, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.058351099491119385, + "fcm_dpo/q_t": 0.4985370635986328, + "grad_norm": 26.187711715698242, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.07552579045295715, + "logits/rejected": 0.05123044550418854, + "logps/chosen": -52.041358947753906, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.82368469238281, + "loss": 1.3824, + "margin_dpo/margin_mean": 0.058351099491119385, + "margin_dpo/margin_std": 0.8890206813812256, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.8652439117431641, + "KL/mean": -1.0923452377319336, + "KL/rejected_KL_mean": -1.3194541931152344, + "KL/std": 0.8883547782897949, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4542124569416046, + "fcm_dpo/q_t": 0.48869743943214417, + "grad_norm": 30.891233444213867, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.1984306275844574, + "logits/rejected": 0.16880089044570923, + "logps/chosen": -68.15393829345703, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.76226043701172, + "loss": 1.3448, + "margin_dpo/margin_mean": 0.4542120397090912, + "margin_dpo/margin_std": 1.1676384210586548, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.9264984130859375, + "KL/mean": -1.099461555480957, + "KL/rejected_KL_mean": -1.2724266052246094, + "KL/std": 0.8448681831359863, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.34592562913894653, + "fcm_dpo/q_t": 0.4913817346096039, + "grad_norm": 29.38481330871582, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.11555850505828857, + "logits/rejected": 0.09126327931880951, + "logps/chosen": -71.66990661621094, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.53742980957031, + "loss": 1.3552, + "margin_dpo/margin_mean": 0.34592553973197937, + "margin_dpo/margin_std": 1.115427017211914, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.7729225158691406, + "KL/mean": -0.9254127144813538, + "KL/rejected_KL_mean": -1.0778999328613281, + "KL/std": 0.7637161612510681, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.30497562885284424, + "fcm_dpo/q_t": 0.49239760637283325, + "grad_norm": 29.26283073425293, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.10749173164367676, + "logits/rejected": 0.05099410563707352, + "logps/chosen": -61.375526428222656, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -76.30026245117188, + "loss": 1.3582, + "margin_dpo/margin_mean": 0.3049759268760681, + "margin_dpo/margin_std": 0.9200187921524048, + "step": 67 + }, + { + "KL/chosen_KL_mean": -1.2506217956542969, + "KL/mean": -1.421552062034607, + "KL/rejected_KL_mean": -1.5924835205078125, + "KL/std": 0.9510532021522522, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.34186407923698425, + "fcm_dpo/q_t": 0.49147123098373413, + "grad_norm": 31.39417266845703, + "learning_rate": 5e-07, + "logits/chosen": 0.06202474609017372, + "logits/rejected": 0.03223402798175812, + "logps/chosen": -78.77899169921875, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.770263671875, + "loss": 1.3563, + "margin_dpo/margin_mean": 0.34186458587646484, + "margin_dpo/margin_std": 1.2264301776885986, + "step": 68 + }, + { + "KL/chosen_KL_mean": -1.184225082397461, + "KL/mean": -1.4764790534973145, + "KL/rejected_KL_mean": -1.7687301635742188, + "KL/std": 1.024730920791626, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5845098495483398, + "fcm_dpo/q_t": 0.4854779541492462, + "grad_norm": 30.998098373413086, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.08792998641729355, + "logits/rejected": 0.045593686401844025, + "logps/chosen": -67.12727355957031, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -91.54228973388672, + "loss": 1.3326, + "margin_dpo/margin_mean": 0.5845092535018921, + "margin_dpo/margin_std": 1.2174975872039795, + "step": 69 + }, + { + "KL/chosen_KL_mean": -1.2389240264892578, + "KL/mean": -1.3959846496582031, + "KL/rejected_KL_mean": -1.5530376434326172, + "KL/std": 0.9079196453094482, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.31411120295524597, + "fcm_dpo/q_t": 0.49217674136161804, + "grad_norm": 28.801374435424805, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.16327008605003357, + "logits/rejected": 0.1396973729133606, + "logps/chosen": -63.19683837890625, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -77.36249542236328, + "loss": 1.3588, + "margin_dpo/margin_mean": 0.3141113817691803, + "margin_dpo/margin_std": 1.214660406112671, + "step": 70 + }, + { + "KL/chosen_KL_mean": -1.4482479095458984, + "KL/mean": -1.5614473819732666, + "KL/rejected_KL_mean": -1.6746501922607422, + "KL/std": 1.0249958038330078, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22640283405780792, + "fcm_dpo/q_t": 0.49432241916656494, + "grad_norm": 28.454469680786133, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.11360330879688263, + "logits/rejected": 0.09736478328704834, + "logps/chosen": -64.79582214355469, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -69.17123413085938, + "loss": 1.3693, + "margin_dpo/margin_mean": 0.22640304267406464, + "margin_dpo/margin_std": 1.474951982498169, + "step": 71 + }, + { + "KL/chosen_KL_mean": -1.1897659301757812, + "KL/mean": -1.553931713104248, + "KL/rejected_KL_mean": -1.9180984497070312, + "KL/std": 1.0742740631103516, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7283300161361694, + "fcm_dpo/q_t": 0.48187729716300964, + "grad_norm": 30.26369857788086, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.1204986423254013, + "logits/rejected": 0.05717724561691284, + "logps/chosen": -57.04906463623047, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -70.37234497070312, + "loss": 1.3191, + "margin_dpo/margin_mean": 0.7283297181129456, + "margin_dpo/margin_std": 1.2839291095733643, + "step": 72 + }, + { + "KL/chosen_KL_mean": -1.8362503051757812, + "KL/mean": -1.8976032733917236, + "KL/rejected_KL_mean": -1.9589519500732422, + "KL/std": 1.2357733249664307, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12270593643188477, + "fcm_dpo/q_t": 0.4970957636833191, + "grad_norm": 31.9867000579834, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.11336830258369446, + "logits/rejected": 0.09915591776371002, + "logps/chosen": -70.97505187988281, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -81.00482177734375, + "loss": 1.3809, + "margin_dpo/margin_mean": 0.12270653247833252, + "margin_dpo/margin_std": 1.5762428045272827, + "step": 73 + }, + { + "KL/chosen_KL_mean": -1.2705440521240234, + "KL/mean": -1.6108319759368896, + "KL/rejected_KL_mean": -1.9511184692382812, + "KL/std": 1.0245332717895508, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6805762648582458, + "fcm_dpo/q_t": 0.48308998346328735, + "grad_norm": 27.936552047729492, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.1540667563676834, + "logits/rejected": 0.09951502084732056, + "logps/chosen": -51.19428253173828, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -83.68325805664062, + "loss": 1.3243, + "margin_dpo/margin_mean": 0.6805763244628906, + "margin_dpo/margin_std": 1.3530315160751343, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.3861274719238281, + "KL/mean": -1.7660695314407349, + "KL/rejected_KL_mean": -2.146007537841797, + "KL/std": 1.1102235317230225, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7598824501037598, + "fcm_dpo/q_t": 0.4811212420463562, + "grad_norm": 26.1428165435791, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.12207367271184921, + "logits/rejected": 0.0893176943063736, + "logps/chosen": -47.45487976074219, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -68.26415252685547, + "loss": 1.3163, + "margin_dpo/margin_mean": 0.7598823308944702, + "margin_dpo/margin_std": 1.3477892875671387, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.5627460479736328, + "KL/mean": -1.734074354171753, + "KL/rejected_KL_mean": -1.9054031372070312, + "KL/std": 1.2022194862365723, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3426612615585327, + "fcm_dpo/q_t": 0.49147289991378784, + "grad_norm": 29.220705032348633, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.12868475914001465, + "logits/rejected": 0.08463678508996964, + "logps/chosen": -55.62549591064453, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -76.78004455566406, + "loss": 1.3586, + "margin_dpo/margin_mean": 0.3426612913608551, + "margin_dpo/margin_std": 1.5809069871902466, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6425457000732422, + "KL/mean": -2.0247294902801514, + "KL/rejected_KL_mean": -2.406909942626953, + "KL/std": 1.2334535121917725, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7643673419952393, + "fcm_dpo/q_t": 0.4810134768486023, + "grad_norm": 29.239410400390625, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.17115390300750732, + "logits/rejected": 0.12893466651439667, + "logps/chosen": -54.71863555908203, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -76.86293029785156, + "loss": 1.3169, + "margin_dpo/margin_mean": 0.7643671631813049, + "margin_dpo/margin_std": 1.4656544923782349, + "step": 77 + }, + { + "KL/chosen_KL_mean": -1.8553199768066406, + "KL/mean": -2.048530340194702, + "KL/rejected_KL_mean": -2.2417449951171875, + "KL/std": 1.2667438983917236, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.38642293214797974, + "fcm_dpo/q_t": 0.4903751313686371, + "grad_norm": 29.433731079101562, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.11265414953231812, + "logits/rejected": 0.09231537580490112, + "logps/chosen": -69.58074188232422, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -81.28101348876953, + "loss": 1.3531, + "margin_dpo/margin_mean": 0.3864227533340454, + "margin_dpo/margin_std": 1.4329341650009155, + "step": 78 + }, + { + "KL/chosen_KL_mean": -1.7077217102050781, + "KL/mean": -2.3071465492248535, + "KL/rejected_KL_mean": -2.9065704345703125, + "KL/std": 1.5384122133255005, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1988508701324463, + "fcm_dpo/q_t": 0.47053754329681396, + "grad_norm": 30.798582077026367, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.18950387835502625, + "logits/rejected": 0.12797753512859344, + "logps/chosen": -53.86836624145508, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -86.21719360351562, + "loss": 1.2795, + "margin_dpo/margin_mean": 1.198850393295288, + "margin_dpo/margin_std": 1.959941029548645, + "step": 79 + }, + { + "KL/chosen_KL_mean": -2.024747848510742, + "KL/mean": -2.3166556358337402, + "KL/rejected_KL_mean": -2.6085662841796875, + "KL/std": 1.369431495666504, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5838208198547363, + "fcm_dpo/q_t": 0.48558056354522705, + "grad_norm": 28.381168365478516, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.14689680933952332, + "logits/rejected": 0.08690305054187775, + "logps/chosen": -63.43531036376953, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -81.26861572265625, + "loss": 1.3383, + "margin_dpo/margin_mean": 0.5838209390640259, + "margin_dpo/margin_std": 1.8922240734100342, + "step": 80 + }, + { + "KL/chosen_KL_mean": -2.1597976684570312, + "KL/mean": -2.5934157371520996, + "KL/rejected_KL_mean": -3.0270309448242188, + "KL/std": 1.434091329574585, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8672326803207397, + "fcm_dpo/q_t": 0.47862696647644043, + "grad_norm": 29.007713317871094, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.13177847862243652, + "logits/rejected": 0.09794065356254578, + "logps/chosen": -65.96417236328125, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -82.37544250488281, + "loss": 1.3097, + "margin_dpo/margin_mean": 0.8672324419021606, + "margin_dpo/margin_std": 1.7289981842041016, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.7621707916259766, + "KL/mean": -2.3001327514648438, + "KL/rejected_KL_mean": -2.8380966186523438, + "KL/std": 1.359740972518921, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.075925588607788, + "fcm_dpo/q_t": 0.4732835590839386, + "grad_norm": 26.572818756103516, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.12116063386201859, + "logits/rejected": 0.057523056864738464, + "logps/chosen": -50.58006286621094, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -73.153076171875, + "loss": 1.2876, + "margin_dpo/margin_mean": 1.0759259462356567, + "margin_dpo/margin_std": 1.5349102020263672, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.0419349670410156, + "KL/mean": -2.62204647064209, + "KL/rejected_KL_mean": -3.202159881591797, + "KL/std": 1.4831633567810059, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1602277755737305, + "fcm_dpo/q_t": 0.47140026092529297, + "grad_norm": 29.156784057617188, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.18060043454170227, + "logits/rejected": 0.13148349523544312, + "logps/chosen": -59.192710876464844, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -78.37318420410156, + "loss": 1.2821, + "margin_dpo/margin_mean": 1.1602275371551514, + "margin_dpo/margin_std": 1.8395249843597412, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.5159435272216797, + "KL/mean": -2.952871322631836, + "KL/rejected_KL_mean": -3.389801025390625, + "KL/std": 1.8310933113098145, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8738585710525513, + "fcm_dpo/q_t": 0.4783720076084137, + "grad_norm": 30.70692253112793, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.16691604256629944, + "logits/rejected": 0.1250467747449875, + "logps/chosen": -67.29324340820312, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -88.10929870605469, + "loss": 1.3101, + "margin_dpo/margin_mean": 0.8738582134246826, + "margin_dpo/margin_std": 1.9217841625213623, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.2303695678710938, + "KL/mean": -2.8911349773406982, + "KL/rejected_KL_mean": -3.5518951416015625, + "KL/std": 1.786381721496582, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3215255737304688, + "fcm_dpo/q_t": 0.46737951040267944, + "grad_norm": 26.60832405090332, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.14956089854240417, + "logits/rejected": 0.10656014084815979, + "logps/chosen": -52.48206329345703, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -70.10629272460938, + "loss": 1.2688, + "margin_dpo/margin_mean": 1.3215255737304688, + "margin_dpo/margin_std": 1.9796284437179565, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.857900619506836, + "KL/mean": -3.3198752403259277, + "KL/rejected_KL_mean": -3.781848907470703, + "KL/std": 1.9224777221679688, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9239518642425537, + "fcm_dpo/q_t": 0.47717535495758057, + "grad_norm": 27.272216796875, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.16544556617736816, + "logits/rejected": 0.147491455078125, + "logps/chosen": -63.58708190917969, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -76.09146118164062, + "loss": 1.3065, + "margin_dpo/margin_mean": 0.9239521026611328, + "margin_dpo/margin_std": 2.0516607761383057, + "step": 86 + }, + { + "KL/chosen_KL_mean": -3.028636932373047, + "KL/mean": -3.547091007232666, + "KL/rejected_KL_mean": -4.065544128417969, + "KL/std": 2.0648305416107178, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0369104146957397, + "fcm_dpo/q_t": 0.4744616746902466, + "grad_norm": 30.57183837890625, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.14023475348949432, + "logits/rejected": 0.08374907076358795, + "logps/chosen": -68.78660583496094, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -88.87713623046875, + "loss": 1.3007, + "margin_dpo/margin_mean": 1.0369102954864502, + "margin_dpo/margin_std": 2.476595640182495, + "step": 87 + }, + { + "KL/chosen_KL_mean": -3.0699501037597656, + "KL/mean": -3.4808826446533203, + "KL/rejected_KL_mean": -3.8918113708496094, + "KL/std": 2.3378701210021973, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8218619227409363, + "fcm_dpo/q_t": 0.4799662232398987, + "grad_norm": 30.38210105895996, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.23296837508678436, + "logits/rejected": 0.2060338854789734, + "logps/chosen": -65.89398193359375, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -78.85252380371094, + "loss": 1.3288, + "margin_dpo/margin_mean": 0.821861982345581, + "margin_dpo/margin_std": 2.9647645950317383, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.5609130859375, + "KL/mean": -3.2582736015319824, + "KL/rejected_KL_mean": -3.9556312561035156, + "KL/std": 2.156515598297119, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3947181701660156, + "fcm_dpo/q_t": 0.4660217761993408, + "grad_norm": 29.258838653564453, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.237514466047287, + "logits/rejected": 0.15626181662082672, + "logps/chosen": -43.752349853515625, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -89.40332794189453, + "loss": 1.2685, + "margin_dpo/margin_mean": 1.3947186470031738, + "margin_dpo/margin_std": 2.536036968231201, + "step": 89 + }, + { + "KL/chosen_KL_mean": -3.1376190185546875, + "KL/mean": -3.7834763526916504, + "KL/rejected_KL_mean": -4.4293365478515625, + "KL/std": 2.46217942237854, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2917141914367676, + "fcm_dpo/q_t": 0.4683051109313965, + "grad_norm": 29.12344741821289, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.168723464012146, + "logits/rejected": 0.11255622655153275, + "logps/chosen": -59.721527099609375, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -91.29911804199219, + "loss": 1.2769, + "margin_dpo/margin_mean": 1.291713833808899, + "margin_dpo/margin_std": 2.4582650661468506, + "step": 90 + }, + { + "KL/chosen_KL_mean": -3.149148941040039, + "KL/mean": -3.722820281982422, + "KL/rejected_KL_mean": -4.296489715576172, + "KL/std": 2.303582191467285, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1473394632339478, + "fcm_dpo/q_t": 0.47223398089408875, + "grad_norm": 25.720582962036133, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.1862848550081253, + "logits/rejected": 0.14085903763771057, + "logps/chosen": -55.531494140625, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -76.47291564941406, + "loss": 1.2941, + "margin_dpo/margin_mean": 1.147339105606079, + "margin_dpo/margin_std": 2.7885167598724365, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.230426788330078, + "KL/mean": -4.01956844329834, + "KL/rejected_KL_mean": -4.808712005615234, + "KL/std": 2.6578006744384766, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.578284740447998, + "fcm_dpo/q_t": 0.4619179964065552, + "grad_norm": 26.956829071044922, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.22094112634658813, + "logits/rejected": 0.1804642379283905, + "logps/chosen": -56.23912811279297, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -84.58683776855469, + "loss": 1.2588, + "margin_dpo/margin_mean": 1.578284502029419, + "margin_dpo/margin_std": 3.120558738708496, + "step": 92 + }, + { + "KL/chosen_KL_mean": -3.615804672241211, + "KL/mean": -4.103872299194336, + "KL/rejected_KL_mean": -4.591936111450195, + "KL/std": 2.629575729370117, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9761362075805664, + "fcm_dpo/q_t": 0.4767376184463501, + "grad_norm": 24.554616928100586, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.20379352569580078, + "logits/rejected": 0.17453384399414062, + "logps/chosen": -48.522857666015625, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -63.37989044189453, + "loss": 1.3171, + "margin_dpo/margin_mean": 0.9761360883712769, + "margin_dpo/margin_std": 3.2695860862731934, + "step": 93 + }, + { + "KL/chosen_KL_mean": -3.2867393493652344, + "KL/mean": -4.221029758453369, + "KL/rejected_KL_mean": -5.1553192138671875, + "KL/std": 3.2257328033447266, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8685789108276367, + "fcm_dpo/q_t": 0.45501282811164856, + "grad_norm": 27.455610275268555, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.2643049359321594, + "logits/rejected": 0.22353461384773254, + "logps/chosen": -63.224510192871094, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -84.46916198730469, + "loss": 1.2467, + "margin_dpo/margin_mean": 1.868578553199768, + "margin_dpo/margin_std": 3.965810537338257, + "step": 94 + }, + { + "KL/chosen_KL_mean": -4.280330657958984, + "KL/mean": -5.080137252807617, + "KL/rejected_KL_mean": -5.87994384765625, + "KL/std": 2.722566604614258, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5996131896972656, + "fcm_dpo/q_t": 0.4613925516605377, + "grad_norm": 29.14002799987793, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.15258441865444183, + "logits/rejected": 0.0883299708366394, + "logps/chosen": -64.44882202148438, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -96.6166000366211, + "loss": 1.2618, + "margin_dpo/margin_mean": 1.5996129512786865, + "margin_dpo/margin_std": 3.4419288635253906, + "step": 95 + }, + { + "KL/chosen_KL_mean": -4.186677932739258, + "KL/mean": -5.113858222961426, + "KL/rejected_KL_mean": -6.041038513183594, + "KL/std": 3.425692081451416, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8543610572814941, + "fcm_dpo/q_t": 0.4570516049861908, + "grad_norm": 27.39577293395996, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.18779154121875763, + "logits/rejected": 0.14556936919689178, + "logps/chosen": -64.85545349121094, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -94.34777069091797, + "loss": 1.2594, + "margin_dpo/margin_mean": 1.854360818862915, + "margin_dpo/margin_std": 4.501690864562988, + "step": 96 + }, + { + "KL/chosen_KL_mean": -4.962148666381836, + "KL/mean": -5.490826606750488, + "KL/rejected_KL_mean": -6.019508361816406, + "KL/std": 3.6154727935791016, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0573649406433105, + "fcm_dpo/q_t": 0.47493094205856323, + "grad_norm": 36.246150970458984, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.12958383560180664, + "logits/rejected": 0.08639142662286758, + "logps/chosen": -70.00627136230469, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -84.44043731689453, + "loss": 1.3422, + "margin_dpo/margin_mean": 1.0573647022247314, + "margin_dpo/margin_std": 4.932512283325195, + "step": 97 + }, + { + "KL/chosen_KL_mean": -4.633974075317383, + "KL/mean": -4.971208095550537, + "KL/rejected_KL_mean": -5.308444976806641, + "KL/std": 3.280994415283203, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6744681596755981, + "fcm_dpo/q_t": 0.4831414818763733, + "grad_norm": 30.31377410888672, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.23023685812950134, + "logits/rejected": 0.2029840350151062, + "logps/chosen": -60.13720703125, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -78.12398529052734, + "loss": 1.3469, + "margin_dpo/margin_mean": 0.6744686365127563, + "margin_dpo/margin_std": 3.2570295333862305, + "step": 98 + }, + { + "KL/chosen_KL_mean": -4.691219329833984, + "KL/mean": -5.205721855163574, + "KL/rejected_KL_mean": -5.7202301025390625, + "KL/std": 3.546834945678711, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0290096998214722, + "fcm_dpo/q_t": 0.4757050573825836, + "grad_norm": 32.59518814086914, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.2541065216064453, + "logits/rejected": 0.21417078375816345, + "logps/chosen": -63.2668571472168, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -84.41383361816406, + "loss": 1.3378, + "margin_dpo/margin_mean": 1.0290095806121826, + "margin_dpo/margin_std": 4.595114707946777, + "step": 99 + }, + { + "KL/chosen_KL_mean": -5.293548583984375, + "KL/mean": -6.042011260986328, + "KL/rejected_KL_mean": -6.790477752685547, + "KL/std": 4.170048236846924, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.496924638748169, + "fcm_dpo/q_t": 0.46660345792770386, + "grad_norm": 32.89773941040039, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.20524325966835022, + "logits/rejected": 0.19783976674079895, + "logps/chosen": -84.87698364257812, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -98.94306182861328, + "loss": 1.3287, + "margin_dpo/margin_mean": 1.4969251155853271, + "margin_dpo/margin_std": 5.857483386993408, + "step": 100 + }, + { + "KL/chosen_KL_mean": -4.28514289855957, + "KL/mean": -5.326085090637207, + "KL/rejected_KL_mean": -6.367027282714844, + "KL/std": 3.696342945098877, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.22752070426940918, + "fcm_dpo/delta": 0.07919151335954666, + "fcm_dpo/margin": 2.0818839073181152, + "fcm_dpo/q_t": 0.4497947096824646, + "grad_norm": 81.98783111572266, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.21574577689170837, + "logits/rejected": 0.17738720774650574, + "logps/chosen": -56.61792755126953, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -75.92292785644531, + "loss": 1.2433, + "margin_dpo/margin_mean": 2.0818839073181152, + "margin_dpo/margin_std": 4.205405235290527, + "step": 101 + }, + { + "KL/chosen_KL_mean": -5.278135299682617, + "KL/mean": -5.710550308227539, + "KL/rejected_KL_mean": -6.142971038818359, + "KL/std": 3.6688332557678223, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.35483452677726746, + "fcm_dpo/delta": -7.2879483923316e-05, + "fcm_dpo/margin": 0.8648367524147034, + "fcm_dpo/q_t": 0.47156286239624023, + "grad_norm": 162.13876342773438, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.13888651132583618, + "logits/rejected": 0.10068652778863907, + "logps/chosen": -70.02162170410156, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -75.20429992675781, + "loss": 1.7198, + "margin_dpo/margin_mean": 0.8648366928100586, + "margin_dpo/margin_std": 5.406614303588867, + "step": 102 + }, + { + "KL/chosen_KL_mean": -4.759641647338867, + "KL/mean": -5.5327959060668945, + "KL/rejected_KL_mean": -6.305950164794922, + "KL/std": 3.811723232269287, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.2257329523563385, + "fcm_dpo/delta": -0.23946459591388702, + "fcm_dpo/margin": 1.5463120937347412, + "fcm_dpo/q_t": 0.3969665467739105, + "grad_norm": 121.77664947509766, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.2512431740760803, + "logits/rejected": 0.2177349030971527, + "logps/chosen": -68.59628295898438, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -85.62957763671875, + "loss": 1.4546, + "margin_dpo/margin_mean": 1.546311855316162, + "margin_dpo/margin_std": 3.6003336906433105, + "step": 103 + }, + { + "KL/chosen_KL_mean": -4.559057235717773, + "KL/mean": -5.69294548034668, + "KL/rejected_KL_mean": -6.826839447021484, + "KL/std": 4.804152011871338, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.2677783966064453, + "fcm_dpo/q_t": 0.49564671516418457, + "grad_norm": 3.038849353790283, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.23869748413562775, + "logits/rejected": 0.15296530723571777, + "logps/chosen": -65.55826568603516, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -105.67329406738281, + "loss": 1.3695, + "margin_dpo/margin_mean": 2.267778158187866, + "margin_dpo/margin_std": 6.126588821411133, + "step": 104 + }, + { + "KL/chosen_KL_mean": -5.151271820068359, + "KL/mean": -6.049022197723389, + "KL/rejected_KL_mean": -6.94677734375, + "KL/std": 3.8394556045532227, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7955060005187988, + "fcm_dpo/q_t": 0.4965510368347168, + "grad_norm": 2.853419303894043, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.17378970980644226, + "logits/rejected": 0.12029208987951279, + "logps/chosen": -76.1015396118164, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -94.83018493652344, + "loss": 1.3729, + "margin_dpo/margin_mean": 1.7955060005187988, + "margin_dpo/margin_std": 4.681982040405273, + "step": 105 + }, + { + "KL/chosen_KL_mean": -4.813375473022461, + "KL/mean": -5.589751243591309, + "KL/rejected_KL_mean": -6.366130828857422, + "KL/std": 3.7389588356018066, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5527547597885132, + "fcm_dpo/q_t": 0.4970164895057678, + "grad_norm": 2.309171438217163, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.2168569266796112, + "logits/rejected": 0.20323088765144348, + "logps/chosen": -67.272705078125, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -73.37207794189453, + "loss": 1.3748, + "margin_dpo/margin_mean": 1.5527546405792236, + "margin_dpo/margin_std": 5.025470733642578, + "step": 106 + }, + { + "KL/chosen_KL_mean": -6.303882598876953, + "KL/mean": -6.7031097412109375, + "KL/rejected_KL_mean": -7.102336883544922, + "KL/std": 4.748565196990967, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7984510064125061, + "fcm_dpo/q_t": 0.49846479296684265, + "grad_norm": 2.6205790042877197, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.19034229218959808, + "logits/rejected": 0.16588297486305237, + "logps/chosen": -82.141845703125, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -94.84272003173828, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.7984509468078613, + "margin_dpo/margin_std": 5.902289390563965, + "step": 107 + }, + { + "KL/chosen_KL_mean": -5.396709442138672, + "KL/mean": -6.335811138153076, + "KL/rejected_KL_mean": -7.274906158447266, + "KL/std": 4.074840545654297, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8781964778900146, + "fcm_dpo/q_t": 0.49639254808425903, + "grad_norm": 2.582226037979126, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.19318030774593353, + "logits/rejected": 0.16133251786231995, + "logps/chosen": -73.78994750976562, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -90.517578125, + "loss": 1.3723, + "margin_dpo/margin_mean": 1.8781968355178833, + "margin_dpo/margin_std": 5.379508972167969, + "step": 108 + }, + { + "KL/chosen_KL_mean": -4.847797393798828, + "KL/mean": -6.345134735107422, + "KL/rejected_KL_mean": -7.842475891113281, + "KL/std": 5.034552574157715, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.994675636291504, + "fcm_dpo/q_t": 0.49424993991851807, + "grad_norm": 2.9282801151275635, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.20009146630764008, + "logits/rejected": 0.14455196261405945, + "logps/chosen": -60.37527847290039, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -91.39466857910156, + "loss": 1.3638, + "margin_dpo/margin_mean": 2.994675636291504, + "margin_dpo/margin_std": 5.442989349365234, + "step": 109 + }, + { + "KL/chosen_KL_mean": -6.40728759765625, + "KL/mean": -6.496090888977051, + "KL/rejected_KL_mean": -6.584896087646484, + "KL/std": 4.201725006103516, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17760717868804932, + "fcm_dpo/q_t": 0.49965718388557434, + "grad_norm": 2.743898630142212, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.16291868686676025, + "logits/rejected": 0.17184513807296753, + "logps/chosen": -87.56603240966797, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -79.14511108398438, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.1776072382926941, + "margin_dpo/margin_std": 5.59630012512207, + "step": 110 + }, + { + "KL/chosen_KL_mean": -5.86181640625, + "KL/mean": -6.897708892822266, + "KL/rejected_KL_mean": -7.933605194091797, + "KL/std": 4.601965427398682, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0717906951904297, + "fcm_dpo/q_t": 0.4960215389728546, + "grad_norm": 2.4579412937164307, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.2789857089519501, + "logits/rejected": 0.24208299815654755, + "logps/chosen": -58.220802307128906, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -84.99510955810547, + "loss": 1.3709, + "margin_dpo/margin_mean": 2.0717906951904297, + "margin_dpo/margin_std": 5.603476047515869, + "step": 111 + }, + { + "KL/chosen_KL_mean": -5.752780914306641, + "KL/mean": -7.0450944900512695, + "KL/rejected_KL_mean": -8.337406158447266, + "KL/std": 4.755733966827393, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.5846307277679443, + "fcm_dpo/q_t": 0.49503466486930847, + "grad_norm": 3.0572729110717773, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.1905369758605957, + "logits/rejected": 0.12637881934642792, + "logps/chosen": -68.77284240722656, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -119.70681762695312, + "loss": 1.3669, + "margin_dpo/margin_mean": 2.584629535675049, + "margin_dpo/margin_std": 5.197779655456543, + "step": 112 + }, + { + "KL/chosen_KL_mean": -5.822654724121094, + "KL/mean": -6.4955949783325195, + "KL/rejected_KL_mean": -7.1685333251953125, + "KL/std": 3.995758533477783, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3458778858184814, + "fcm_dpo/q_t": 0.49741414189338684, + "grad_norm": 2.6555185317993164, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.2019399255514145, + "logits/rejected": 0.16312995553016663, + "logps/chosen": -61.63031768798828, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -77.00867462158203, + "loss": 1.3764, + "margin_dpo/margin_mean": 1.3458775281906128, + "margin_dpo/margin_std": 5.301340103149414, + "step": 113 + }, + { + "KL/chosen_KL_mean": -5.752845764160156, + "KL/mean": -6.942575454711914, + "KL/rejected_KL_mean": -8.132308959960938, + "KL/std": 4.493397235870361, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.3794596195220947, + "fcm_dpo/q_t": 0.495430052280426, + "grad_norm": 2.5707952976226807, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.26456063985824585, + "logits/rejected": 0.20437636971473694, + "logps/chosen": -72.08561706542969, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -79.74720764160156, + "loss": 1.3685, + "margin_dpo/margin_mean": 2.3794591426849365, + "margin_dpo/margin_std": 5.449493885040283, + "step": 114 + }, + { + "KL/chosen_KL_mean": -6.265972137451172, + "KL/mean": -7.2780961990356445, + "KL/rejected_KL_mean": -8.290224075317383, + "KL/std": 4.36235237121582, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0242466926574707, + "fcm_dpo/q_t": 0.49611204862594604, + "grad_norm": 2.4445128440856934, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.2416526824235916, + "logits/rejected": 0.17938242852687836, + "logps/chosen": -62.015010833740234, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -87.88871765136719, + "loss": 1.3712, + "margin_dpo/margin_mean": 2.0242466926574707, + "margin_dpo/margin_std": 5.242649078369141, + "step": 115 + }, + { + "KL/chosen_KL_mean": -6.046346664428711, + "KL/mean": -7.712983131408691, + "KL/rejected_KL_mean": -9.379619598388672, + "KL/std": 4.501278877258301, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.3332784175872803, + "fcm_dpo/q_t": 0.49359679222106934, + "grad_norm": 2.852839708328247, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.2487809658050537, + "logits/rejected": 0.1809816211462021, + "logps/chosen": -55.41151428222656, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -82.22632598876953, + "loss": 1.3612, + "margin_dpo/margin_mean": 3.3332786560058594, + "margin_dpo/margin_std": 5.188892364501953, + "step": 116 + }, + { + "KL/chosen_KL_mean": -6.292646408081055, + "KL/mean": -7.328824043273926, + "KL/rejected_KL_mean": -8.365001678466797, + "KL/std": 4.449042320251465, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0723490715026855, + "fcm_dpo/q_t": 0.496019184589386, + "grad_norm": 2.2913970947265625, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.26744014024734497, + "logits/rejected": 0.23043423891067505, + "logps/chosen": -64.0035400390625, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -78.13754272460938, + "loss": 1.371, + "margin_dpo/margin_mean": 2.0723495483398438, + "margin_dpo/margin_std": 5.764100551605225, + "step": 117 + }, + { + "KL/chosen_KL_mean": -6.316926956176758, + "KL/mean": -7.789030075073242, + "KL/rejected_KL_mean": -9.261127471923828, + "KL/std": 5.045524597167969, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.9442031383514404, + "fcm_dpo/q_t": 0.4943440854549408, + "grad_norm": 2.639897346496582, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.33026766777038574, + "logits/rejected": 0.28865814208984375, + "logps/chosen": -58.79682540893555, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -90.62025451660156, + "loss": 1.3642, + "margin_dpo/margin_mean": 2.9442026615142822, + "margin_dpo/margin_std": 5.081968307495117, + "step": 118 + }, + { + "KL/chosen_KL_mean": -8.139110565185547, + "KL/mean": -9.161100387573242, + "KL/rejected_KL_mean": -10.183094024658203, + "KL/std": 6.225261688232422, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0439817905426025, + "fcm_dpo/q_t": 0.4960786998271942, + "grad_norm": 2.5201234817504883, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.2730048894882202, + "logits/rejected": 0.2327132672071457, + "logps/chosen": -69.49678039550781, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -85.898193359375, + "loss": 1.3716, + "margin_dpo/margin_mean": 2.0439815521240234, + "margin_dpo/margin_std": 7.831289768218994, + "step": 119 + }, + { + "KL/chosen_KL_mean": -6.7714385986328125, + "KL/mean": -8.515937805175781, + "KL/rejected_KL_mean": -10.260440826416016, + "KL/std": 5.0966644287109375, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.4890010356903076, + "fcm_dpo/q_t": 0.49329859018325806, + "grad_norm": 2.945573091506958, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.23838064074516296, + "logits/rejected": 0.17867352068424225, + "logps/chosen": -66.67900848388672, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -89.95150756835938, + "loss": 1.3601, + "margin_dpo/margin_mean": 3.4890012741088867, + "margin_dpo/margin_std": 5.204916000366211, + "step": 120 + }, + { + "KL/chosen_KL_mean": -7.601043701171875, + "KL/mean": -9.675811767578125, + "KL/rejected_KL_mean": -11.75058364868164, + "KL/std": 6.724109649658203, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.149538516998291, + "fcm_dpo/q_t": 0.49204063415527344, + "grad_norm": 3.3024706840515137, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.35299158096313477, + "logits/rejected": 0.26683202385902405, + "logps/chosen": -63.267086029052734, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -113.31292724609375, + "loss": 1.3556, + "margin_dpo/margin_mean": 4.149538516998291, + "margin_dpo/margin_std": 7.892666816711426, + "step": 121 + }, + { + "KL/chosen_KL_mean": -8.613723754882812, + "KL/mean": -10.285037994384766, + "KL/rejected_KL_mean": -11.956352233886719, + "KL/std": 7.303105354309082, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.3426246643066406, + "fcm_dpo/q_t": 0.49359560012817383, + "grad_norm": 2.7032148838043213, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.26826468110084534, + "logits/rejected": 0.2332470566034317, + "logps/chosen": -71.9480972290039, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -85.631591796875, + "loss": 1.362, + "margin_dpo/margin_mean": 3.3426246643066406, + "margin_dpo/margin_std": 9.229589462280273, + "step": 122 + }, + { + "KL/chosen_KL_mean": -8.551752090454102, + "KL/mean": -10.23425579071045, + "KL/rejected_KL_mean": -11.916763305664062, + "KL/std": 6.5701446533203125, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.3650121688842773, + "fcm_dpo/q_t": 0.4935457706451416, + "grad_norm": 2.6148831844329834, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.3093302845954895, + "logits/rejected": 0.2672804594039917, + "logps/chosen": -65.45049285888672, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -90.88704681396484, + "loss": 1.3614, + "margin_dpo/margin_mean": 3.3650124073028564, + "margin_dpo/margin_std": 7.255990505218506, + "step": 123 + }, + { + "KL/chosen_KL_mean": -9.072261810302734, + "KL/mean": -11.368237495422363, + "KL/rejected_KL_mean": -13.664215087890625, + "KL/std": 7.325522422790527, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.591949939727783, + "fcm_dpo/q_t": 0.491183876991272, + "grad_norm": 2.9130699634552, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.3153066039085388, + "logits/rejected": 0.25329259037971497, + "logps/chosen": -66.18834686279297, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -101.59495544433594, + "loss": 1.3524, + "margin_dpo/margin_mean": 4.591949462890625, + "margin_dpo/margin_std": 8.780609130859375, + "step": 124 + }, + { + "KL/chosen_KL_mean": -10.442159652709961, + "KL/mean": -12.644015312194824, + "KL/rejected_KL_mean": -14.845863342285156, + "KL/std": 8.506006240844727, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.4036970138549805, + "fcm_dpo/q_t": 0.4915594458580017, + "grad_norm": 3.0990495681762695, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.34969115257263184, + "logits/rejected": 0.28866562247276306, + "logps/chosen": -76.1483383178711, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -106.57298278808594, + "loss": 1.3542, + "margin_dpo/margin_mean": 4.4036970138549805, + "margin_dpo/margin_std": 9.770685195922852, + "step": 125 + }, + { + "KL/chosen_KL_mean": -10.139768600463867, + "KL/mean": -10.864148139953613, + "KL/rejected_KL_mean": -11.588525772094727, + "KL/std": 6.056389808654785, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4487552642822266, + "fcm_dpo/q_t": 0.4972168207168579, + "grad_norm": 2.5714292526245117, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.16045671701431274, + "logits/rejected": 0.15517394244670868, + "logps/chosen": -78.31585693359375, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -76.7060775756836, + "loss": 1.3758, + "margin_dpo/margin_mean": 1.4487552642822266, + "margin_dpo/margin_std": 6.551729679107666, + "step": 126 + }, + { + "KL/chosen_KL_mean": -10.118179321289062, + "KL/mean": -12.15770149230957, + "KL/rejected_KL_mean": -14.197227478027344, + "KL/std": 5.957196235656738, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.07904577255249, + "fcm_dpo/q_t": 0.49216848611831665, + "grad_norm": 2.511080265045166, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.27716439962387085, + "logits/rejected": 0.2475191354751587, + "logps/chosen": -71.9984130859375, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -82.6573486328125, + "loss": 1.356, + "margin_dpo/margin_mean": 4.07904577255249, + "margin_dpo/margin_std": 7.116820812225342, + "step": 127 + }, + { + "KL/chosen_KL_mean": -11.079221725463867, + "KL/mean": -14.211316108703613, + "KL/rejected_KL_mean": -17.343414306640625, + "KL/std": 9.948829650878906, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.264186859130859, + "fcm_dpo/q_t": 0.4880082905292511, + "grad_norm": 3.778376817703247, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.3209880590438843, + "logits/rejected": 0.2622468173503876, + "logps/chosen": -77.7882080078125, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -112.32310485839844, + "loss": 1.3411, + "margin_dpo/margin_mean": 6.264187812805176, + "margin_dpo/margin_std": 12.51683235168457, + "step": 128 + }, + { + "KL/chosen_KL_mean": -13.11799430847168, + "KL/mean": -14.13327407836914, + "KL/rejected_KL_mean": -15.148551940917969, + "KL/std": 9.112518310546875, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0305609703063965, + "fcm_dpo/q_t": 0.49610602855682373, + "grad_norm": 2.6070330142974854, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.30411797761917114, + "logits/rejected": 0.2988065481185913, + "logps/chosen": -78.45681762695312, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -83.20964050292969, + "loss": 1.3727, + "margin_dpo/margin_mean": 2.0305609703063965, + "margin_dpo/margin_std": 11.41251277923584, + "step": 129 + }, + { + "KL/chosen_KL_mean": -12.702459335327148, + "KL/mean": -15.399238586425781, + "KL/rejected_KL_mean": -18.09601593017578, + "KL/std": 9.932580947875977, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.393557548522949, + "fcm_dpo/q_t": 0.4896560311317444, + "grad_norm": 3.132603883743286, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.26491034030914307, + "logits/rejected": 0.21366286277770996, + "logps/chosen": -71.36320495605469, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -97.34111785888672, + "loss": 1.3468, + "margin_dpo/margin_mean": 5.393556594848633, + "margin_dpo/margin_std": 10.183626174926758, + "step": 130 + }, + { + "KL/chosen_KL_mean": -11.906011581420898, + "KL/mean": -15.45162582397461, + "KL/rejected_KL_mean": -18.997241973876953, + "KL/std": 10.712788581848145, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.091231346130371, + "fcm_dpo/q_t": 0.4864232838153839, + "grad_norm": 3.4537715911865234, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.28533366322517395, + "logits/rejected": 0.2199201136827469, + "logps/chosen": -64.42054748535156, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -104.18023681640625, + "loss": 1.3349, + "margin_dpo/margin_mean": 7.091231346130371, + "margin_dpo/margin_std": 12.101551055908203, + "step": 131 + }, + { + "KL/chosen_KL_mean": -13.843952178955078, + "KL/mean": -16.509641647338867, + "KL/rejected_KL_mean": -19.175325393676758, + "KL/std": 11.187821388244629, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.331372261047363, + "fcm_dpo/q_t": 0.4897724986076355, + "grad_norm": 2.8171606063842773, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.3724517822265625, + "logits/rejected": 0.341764897108078, + "logps/chosen": -79.52908325195312, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -88.71653747558594, + "loss": 1.3473, + "margin_dpo/margin_mean": 5.331372261047363, + "margin_dpo/margin_std": 10.194868087768555, + "step": 132 + }, + { + "KL/chosen_KL_mean": -15.300018310546875, + "KL/mean": -16.778060913085938, + "KL/rejected_KL_mean": -18.25609588623047, + "KL/std": 9.998517990112305, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.956071376800537, + "fcm_dpo/q_t": 0.4943294823169708, + "grad_norm": 2.656231164932251, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.2910245954990387, + "logits/rejected": 0.28055471181869507, + "logps/chosen": -78.89813232421875, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -91.98407745361328, + "loss": 1.3658, + "margin_dpo/margin_mean": 2.956071376800537, + "margin_dpo/margin_std": 11.712414741516113, + "step": 133 + }, + { + "KL/chosen_KL_mean": -14.085258483886719, + "KL/mean": -17.977962493896484, + "KL/rejected_KL_mean": -21.87066650390625, + "KL/std": 12.072915077209473, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.785408973693848, + "fcm_dpo/q_t": 0.4851023554801941, + "grad_norm": 3.1521494388580322, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.41877007484436035, + "logits/rejected": 0.36156511306762695, + "logps/chosen": -67.87982940673828, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -96.0380859375, + "loss": 1.3303, + "margin_dpo/margin_mean": 7.785409927368164, + "margin_dpo/margin_std": 14.063751220703125, + "step": 134 + }, + { + "KL/chosen_KL_mean": -14.210046768188477, + "KL/mean": -16.44684410095215, + "KL/rejected_KL_mean": -18.683643341064453, + "KL/std": 10.793325424194336, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.473599910736084, + "fcm_dpo/q_t": 0.4914175271987915, + "grad_norm": 2.542938232421875, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.3599017262458801, + "logits/rejected": 0.32789885997772217, + "logps/chosen": -63.651123046875, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -84.65242004394531, + "loss": 1.354, + "margin_dpo/margin_mean": 4.473599433898926, + "margin_dpo/margin_std": 11.142325401306152, + "step": 135 + }, + { + "KL/chosen_KL_mean": -18.304481506347656, + "KL/mean": -21.979602813720703, + "KL/rejected_KL_mean": -25.654720306396484, + "KL/std": 14.195917129516602, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.350245475769043, + "fcm_dpo/q_t": 0.4859556555747986, + "grad_norm": 3.6188459396362305, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.35441240668296814, + "logits/rejected": 0.27984628081321716, + "logps/chosen": -85.0637435913086, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -120.27259826660156, + "loss": 1.3349, + "margin_dpo/margin_mean": 7.350246429443359, + "margin_dpo/margin_std": 16.540786743164062, + "step": 136 + }, + { + "KL/chosen_KL_mean": -16.800689697265625, + "KL/mean": -20.018104553222656, + "KL/rejected_KL_mean": -23.235515594482422, + "KL/std": 12.882331848144531, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.434828281402588, + "fcm_dpo/q_t": 0.48770397901535034, + "grad_norm": 2.851135492324829, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.39496833086013794, + "logits/rejected": 0.3645436465740204, + "logps/chosen": -73.58448791503906, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -93.13504028320312, + "loss": 1.3409, + "margin_dpo/margin_mean": 6.434828758239746, + "margin_dpo/margin_std": 14.978425979614258, + "step": 137 + }, + { + "KL/chosen_KL_mean": -20.657804489135742, + "KL/mean": -23.541412353515625, + "KL/rejected_KL_mean": -26.425025939941406, + "KL/std": 14.655818939208984, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.767213821411133, + "fcm_dpo/q_t": 0.48898112773895264, + "grad_norm": 2.8203554153442383, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.3970043361186981, + "logits/rejected": 0.36176303029060364, + "logps/chosen": -79.42381286621094, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -94.54874420166016, + "loss": 1.347, + "margin_dpo/margin_mean": 5.767213821411133, + "margin_dpo/margin_std": 17.624059677124023, + "step": 138 + }, + { + "KL/chosen_KL_mean": -22.82883071899414, + "KL/mean": -25.602848052978516, + "KL/rejected_KL_mean": -28.376876831054688, + "KL/std": 12.521333694458008, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.548043727874756, + "fcm_dpo/q_t": 0.48936471343040466, + "grad_norm": 3.1577208042144775, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.3622744083404541, + "logits/rejected": 0.3314181864261627, + "logps/chosen": -94.0544204711914, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -110.56034851074219, + "loss": 1.3473, + "margin_dpo/margin_mean": 5.548043727874756, + "margin_dpo/margin_std": 14.130794525146484, + "step": 139 + }, + { + "KL/chosen_KL_mean": -21.173185348510742, + "KL/mean": -25.737133026123047, + "KL/rejected_KL_mean": -30.30107879638672, + "KL/std": 16.705089569091797, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.127891540527344, + "fcm_dpo/q_t": 0.4825606942176819, + "grad_norm": 3.3629658222198486, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.35938477516174316, + "logits/rejected": 0.3098704516887665, + "logps/chosen": -84.45085144042969, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -113.60755157470703, + "loss": 1.324, + "margin_dpo/margin_mean": 9.127891540527344, + "margin_dpo/margin_std": 21.27337646484375, + "step": 140 + }, + { + "KL/chosen_KL_mean": -23.774330139160156, + "KL/mean": -28.25469970703125, + "KL/rejected_KL_mean": -32.73507308959961, + "KL/std": 16.478158950805664, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.960737228393555, + "fcm_dpo/q_t": 0.48290276527404785, + "grad_norm": 3.510462999343872, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.43517443537712097, + "logits/rejected": 0.38708174228668213, + "logps/chosen": -85.54109954833984, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -121.34109497070312, + "loss": 1.3247, + "margin_dpo/margin_mean": 8.960738182067871, + "margin_dpo/margin_std": 19.960668563842773, + "step": 141 + }, + { + "KL/chosen_KL_mean": -23.207942962646484, + "KL/mean": -26.78273582458496, + "KL/rejected_KL_mean": -30.357532501220703, + "KL/std": 14.630638122558594, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.149584770202637, + "fcm_dpo/q_t": 0.4863271713256836, + "grad_norm": 3.2684221267700195, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.425723671913147, + "logits/rejected": 0.40205299854278564, + "logps/chosen": -88.48271942138672, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -111.49534606933594, + "loss": 1.3353, + "margin_dpo/margin_mean": 7.149584770202637, + "margin_dpo/margin_std": 14.595829010009766, + "step": 142 + }, + { + "KL/chosen_KL_mean": -25.313261032104492, + "KL/mean": -27.440980911254883, + "KL/rejected_KL_mean": -29.56869888305664, + "KL/std": 18.696537017822266, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.255438327789307, + "fcm_dpo/q_t": 0.49176695942878723, + "grad_norm": 3.4273107051849365, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.4723702371120453, + "logits/rejected": 0.4537476897239685, + "logps/chosen": -87.93109130859375, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -99.96109008789062, + "loss": 1.3612, + "margin_dpo/margin_mean": 4.255438327789307, + "margin_dpo/margin_std": 22.319290161132812, + "step": 143 + }, + { + "KL/chosen_KL_mean": -26.418071746826172, + "KL/mean": -30.407001495361328, + "KL/rejected_KL_mean": -34.39592361450195, + "KL/std": 20.060935974121094, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.977849006652832, + "fcm_dpo/q_t": 0.4849676489830017, + "grad_norm": 3.241164207458496, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.4205203056335449, + "logits/rejected": 0.38700443506240845, + "logps/chosen": -87.22076416015625, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -113.46876525878906, + "loss": 1.3353, + "margin_dpo/margin_mean": 7.977849006652832, + "margin_dpo/margin_std": 25.233558654785156, + "step": 144 + }, + { + "KL/chosen_KL_mean": -27.356273651123047, + "KL/mean": -31.281274795532227, + "KL/rejected_KL_mean": -35.206275939941406, + "KL/std": 23.064613342285156, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.850009918212891, + "fcm_dpo/q_t": 0.48525863885879517, + "grad_norm": 3.983025074005127, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.4053303003311157, + "logits/rejected": 0.4174395799636841, + "logps/chosen": -101.96773529052734, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -118.45088958740234, + "loss": 1.3407, + "margin_dpo/margin_mean": 7.850009441375732, + "margin_dpo/margin_std": 30.250947952270508, + "step": 145 + }, + { + "KL/chosen_KL_mean": -26.02267837524414, + "KL/mean": -29.3727970123291, + "KL/rejected_KL_mean": -32.72291946411133, + "KL/std": 19.601165771484375, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.700242519378662, + "fcm_dpo/q_t": 0.4872305989265442, + "grad_norm": 3.2747786045074463, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.4032752215862274, + "logits/rejected": 0.39034542441368103, + "logps/chosen": -83.86366271972656, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -100.1971435546875, + "loss": 1.3418, + "margin_dpo/margin_mean": 6.70024299621582, + "margin_dpo/margin_std": 20.67666244506836, + "step": 146 + }, + { + "KL/chosen_KL_mean": -29.016862869262695, + "KL/mean": -32.25214385986328, + "KL/rejected_KL_mean": -35.48743438720703, + "KL/std": 21.1727237701416, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.470568656921387, + "fcm_dpo/q_t": 0.4874805808067322, + "grad_norm": 3.599782943725586, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.414762020111084, + "logits/rejected": 0.3716977834701538, + "logps/chosen": -95.83033752441406, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -116.66712951660156, + "loss": 1.3455, + "margin_dpo/margin_mean": 6.470569610595703, + "margin_dpo/margin_std": 23.536331176757812, + "step": 147 + }, + { + "KL/chosen_KL_mean": -24.12586212158203, + "KL/mean": -30.170164108276367, + "KL/rejected_KL_mean": -36.21445846557617, + "KL/std": 20.31801414489746, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.088604927062988, + "fcm_dpo/q_t": 0.4769914448261261, + "grad_norm": 3.680696487426758, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.5683670043945312, + "logits/rejected": 0.49711206555366516, + "logps/chosen": -72.81361389160156, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -103.71949768066406, + "loss": 1.3031, + "margin_dpo/margin_mean": 12.088605880737305, + "margin_dpo/margin_std": 22.556541442871094, + "step": 148 + }, + { + "KL/chosen_KL_mean": -29.621259689331055, + "KL/mean": -35.046180725097656, + "KL/rejected_KL_mean": -40.471099853515625, + "KL/std": 22.03968620300293, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.849838256835938, + "fcm_dpo/q_t": 0.47930651903152466, + "grad_norm": 3.6473169326782227, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.46499621868133545, + "logits/rejected": 0.4208984971046448, + "logps/chosen": -84.7650375366211, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -105.26998901367188, + "loss": 1.3114, + "margin_dpo/margin_mean": 10.849838256835938, + "margin_dpo/margin_std": 21.40958595275879, + "step": 149 + }, + { + "KL/chosen_KL_mean": -29.565271377563477, + "KL/mean": -36.95281219482422, + "KL/rejected_KL_mean": -44.34034729003906, + "KL/std": 23.726795196533203, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.775071144104004, + "fcm_dpo/q_t": 0.47204089164733887, + "grad_norm": 3.51343035697937, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.4633652865886688, + "logits/rejected": 0.4186779260635376, + "logps/chosen": -97.03601837158203, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -133.55206298828125, + "loss": 1.2891, + "margin_dpo/margin_mean": 14.775071144104004, + "margin_dpo/margin_std": 29.063199996948242, + "step": 150 + }, + { + "KL/chosen_KL_mean": -21.63004493713379, + "KL/mean": -31.206939697265625, + "KL/rejected_KL_mean": -40.78384017944336, + "KL/std": 26.35299301147461, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.153793334960938, + "fcm_dpo/q_t": 0.46416527032852173, + "grad_norm": 4.2144775390625, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.4840549826622009, + "logits/rejected": 0.4279939532279968, + "logps/chosen": -74.08958435058594, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -119.84684753417969, + "loss": 1.2569, + "margin_dpo/margin_mean": 19.153793334960938, + "margin_dpo/margin_std": 28.657949447631836, + "step": 151 + }, + { + "KL/chosen_KL_mean": -28.358936309814453, + "KL/mean": -34.87946701049805, + "KL/rejected_KL_mean": -41.39999771118164, + "KL/std": 23.711170196533203, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.041062355041504, + "fcm_dpo/q_t": 0.47520697116851807, + "grad_norm": 3.737586498260498, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.5636520981788635, + "logits/rejected": 0.5086890459060669, + "logps/chosen": -84.91276550292969, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -117.95074462890625, + "loss": 1.3005, + "margin_dpo/margin_mean": 13.041061401367188, + "margin_dpo/margin_std": 28.46988868713379, + "step": 152 + }, + { + "KL/chosen_KL_mean": -30.26366424560547, + "KL/mean": -36.74250030517578, + "KL/rejected_KL_mean": -43.22133255004883, + "KL/std": 24.817947387695312, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.957663536071777, + "fcm_dpo/q_t": 0.4754427969455719, + "grad_norm": 3.9588053226470947, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.48824167251586914, + "logits/rejected": 0.4558285176753998, + "logps/chosen": -98.27055358886719, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -118.05615234375, + "loss": 1.2993, + "margin_dpo/margin_mean": 12.957662582397461, + "margin_dpo/margin_std": 25.99228286743164, + "step": 153 + }, + { + "KL/chosen_KL_mean": -29.722576141357422, + "KL/mean": -34.363807678222656, + "KL/rejected_KL_mean": -39.00503921508789, + "KL/std": 29.986373901367188, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.282459259033203, + "fcm_dpo/q_t": 0.4823741316795349, + "grad_norm": 3.928870677947998, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.5707046389579773, + "logits/rejected": 0.5677356719970703, + "logps/chosen": -88.94511413574219, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -103.19635772705078, + "loss": 1.3314, + "margin_dpo/margin_mean": 9.282459259033203, + "margin_dpo/margin_std": 31.800521850585938, + "step": 154 + }, + { + "KL/chosen_KL_mean": -33.82355499267578, + "KL/mean": -41.13679122924805, + "KL/rejected_KL_mean": -48.45003128051758, + "KL/std": 29.099594116210938, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.62647819519043, + "fcm_dpo/q_t": 0.4723411202430725, + "grad_norm": 4.301799297332764, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.580405592918396, + "logits/rejected": 0.5614101886749268, + "logps/chosen": -102.27825164794922, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -126.36766815185547, + "loss": 1.296, + "margin_dpo/margin_mean": 14.626476287841797, + "margin_dpo/margin_std": 35.91053009033203, + "step": 155 + }, + { + "KL/chosen_KL_mean": -34.29534149169922, + "KL/mean": -44.44483184814453, + "KL/rejected_KL_mean": -54.594322204589844, + "KL/std": 32.78551483154297, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 20.29897689819336, + "fcm_dpo/q_t": 0.4618450999259949, + "grad_norm": 4.438568115234375, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.5352226495742798, + "logits/rejected": 0.49391427636146545, + "logps/chosen": -101.56494140625, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -141.553466796875, + "loss": 1.2513, + "margin_dpo/margin_mean": 20.29897689819336, + "margin_dpo/margin_std": 31.937244415283203, + "step": 156 + }, + { + "KL/chosen_KL_mean": -28.4747371673584, + "KL/mean": -34.87397003173828, + "KL/rejected_KL_mean": -41.2732048034668, + "KL/std": 27.206077575683594, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.798463821411133, + "fcm_dpo/q_t": 0.4756951630115509, + "grad_norm": 4.1154913902282715, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.5524776577949524, + "logits/rejected": 0.5371452569961548, + "logps/chosen": -83.24761199951172, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -105.15187072753906, + "loss": 1.3008, + "margin_dpo/margin_mean": 12.798463821411133, + "margin_dpo/margin_std": 25.809242248535156, + "step": 157 + }, + { + "KL/chosen_KL_mean": -34.29819107055664, + "KL/mean": -42.93422317504883, + "KL/rejected_KL_mean": -51.570255279541016, + "KL/std": 35.774105072021484, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 17.272056579589844, + "fcm_dpo/q_t": 0.46861717104911804, + "grad_norm": 4.501484394073486, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.5918360948562622, + "logits/rejected": 0.5534389019012451, + "logps/chosen": -99.22090911865234, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -133.8081512451172, + "loss": 1.286, + "margin_dpo/margin_mean": 17.272056579589844, + "margin_dpo/margin_std": 43.68254089355469, + "step": 158 + }, + { + "KL/chosen_KL_mean": -33.22077178955078, + "KL/mean": -41.89005661010742, + "KL/rejected_KL_mean": -50.559349060058594, + "KL/std": 29.38928985595703, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 17.338571548461914, + "fcm_dpo/q_t": 0.46729815006256104, + "grad_norm": 4.624864101409912, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.5776375532150269, + "logits/rejected": 0.5369961857795715, + "logps/chosen": -90.26776123046875, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -123.8837661743164, + "loss": 1.2746, + "margin_dpo/margin_mean": 17.338571548461914, + "margin_dpo/margin_std": 34.329891204833984, + "step": 159 + }, + { + "KL/chosen_KL_mean": -34.37767791748047, + "KL/mean": -43.98714065551758, + "KL/rejected_KL_mean": -53.59660339355469, + "KL/std": 33.58818054199219, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.21892547607422, + "fcm_dpo/q_t": 0.46424031257629395, + "grad_norm": 5.148886203765869, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.5072331428527832, + "logits/rejected": 0.4576966464519501, + "logps/chosen": -84.1845932006836, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -121.9336166381836, + "loss": 1.265, + "margin_dpo/margin_mean": 19.21892547607422, + "margin_dpo/margin_std": 37.81275177001953, + "step": 160 + }, + { + "KL/chosen_KL_mean": -35.280982971191406, + "KL/mean": -45.04498291015625, + "KL/rejected_KL_mean": -54.808990478515625, + "KL/std": 29.070030212402344, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.007687800098210573, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.527999877929688, + "fcm_dpo/q_t": 0.46312597393989563, + "grad_norm": 5.976346015930176, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.5991806983947754, + "logits/rejected": 0.5647536516189575, + "logps/chosen": -87.7814712524414, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -120.8543930053711, + "loss": 1.2569, + "margin_dpo/margin_mean": 19.52800178527832, + "margin_dpo/margin_std": 31.474956512451172, + "step": 161 + }, + { + "KL/chosen_KL_mean": -45.48345947265625, + "KL/mean": -58.5546989440918, + "KL/rejected_KL_mean": -71.62594604492188, + "KL/std": 45.28847885131836, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.019494613632559776, + "fcm_dpo/delta": 0.08775182068347931, + "fcm_dpo/margin": 26.142498016357422, + "fcm_dpo/q_t": 0.45138120651245117, + "grad_norm": 21.5263614654541, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.6398231983184814, + "logits/rejected": 0.5949274301528931, + "logps/chosen": -114.95265197753906, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -163.63546752929688, + "loss": 1.1634, + "margin_dpo/margin_mean": 26.142494201660156, + "margin_dpo/margin_std": 50.92291259765625, + "step": 162 + }, + { + "KL/chosen_KL_mean": -32.73987579345703, + "KL/mean": -46.96985626220703, + "KL/rejected_KL_mean": -61.1998405456543, + "KL/std": 35.424747467041016, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": -0.12022696435451508, + "fcm_dpo/margin": 28.459957122802734, + "fcm_dpo/q_t": 0.4269000291824341, + "grad_norm": 3.908928155899048, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.6149941682815552, + "logits/rejected": 0.5667312145233154, + "logps/chosen": -83.35371398925781, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -135.82015991210938, + "loss": 1.2688, + "margin_dpo/margin_mean": 28.459957122802734, + "margin_dpo/margin_std": 38.75524139404297, + "step": 163 + }, + { + "KL/chosen_KL_mean": -39.49297332763672, + "KL/mean": -53.75981140136719, + "KL/rejected_KL_mean": -68.02665710449219, + "KL/std": 41.714744567871094, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.533679962158203, + "fcm_dpo/q_t": 0.46781498193740845, + "grad_norm": 4.191322326660156, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.5729007720947266, + "logits/rejected": 0.5150625705718994, + "logps/chosen": -94.34107971191406, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -147.08966064453125, + "loss": 1.2704, + "margin_dpo/margin_mean": 28.53368377685547, + "margin_dpo/margin_std": 42.69341278076172, + "step": 164 + }, + { + "KL/chosen_KL_mean": -45.441650390625, + "KL/mean": -58.43135070800781, + "KL/rejected_KL_mean": -71.42105102539062, + "KL/std": 41.49935531616211, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.979393005371094, + "fcm_dpo/q_t": 0.4706171154975891, + "grad_norm": 4.249202251434326, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.6801586151123047, + "logits/rejected": 0.6297128200531006, + "logps/chosen": -96.5308609008789, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -142.65475463867188, + "loss": 1.2815, + "margin_dpo/margin_mean": 25.979393005371094, + "margin_dpo/margin_std": 44.35696029663086, + "step": 165 + }, + { + "KL/chosen_KL_mean": -49.10633850097656, + "KL/mean": -61.71703338623047, + "KL/rejected_KL_mean": -74.3277359008789, + "KL/std": 45.09733581542969, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.221389770507812, + "fcm_dpo/q_t": 0.47154539823532104, + "grad_norm": 4.738693714141846, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.6274275779724121, + "logits/rejected": 0.5359787940979004, + "logps/chosen": -112.29714965820312, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -168.16793823242188, + "loss": 1.2842, + "margin_dpo/margin_mean": 25.221389770507812, + "margin_dpo/margin_std": 43.412750244140625, + "step": 166 + }, + { + "KL/chosen_KL_mean": -42.655426025390625, + "KL/mean": -56.4607048034668, + "KL/rejected_KL_mean": -70.26599884033203, + "KL/std": 39.476051330566406, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.610572814941406, + "fcm_dpo/q_t": 0.4687613248825073, + "grad_norm": 5.55941915512085, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.6002979278564453, + "logits/rejected": 0.5730512142181396, + "logps/chosen": -101.57969665527344, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -143.23977661132812, + "loss": 1.2729, + "margin_dpo/margin_mean": 27.610572814941406, + "margin_dpo/margin_std": 41.076995849609375, + "step": 167 + }, + { + "KL/chosen_KL_mean": -52.67481231689453, + "KL/mean": -66.61177062988281, + "KL/rejected_KL_mean": -80.54873657226562, + "KL/std": 47.3160400390625, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.873924255371094, + "fcm_dpo/q_t": 0.4689233899116516, + "grad_norm": 4.620728492736816, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.6193612813949585, + "logits/rejected": 0.5847088098526001, + "logps/chosen": -118.3261947631836, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -160.26292419433594, + "loss": 1.2779, + "margin_dpo/margin_mean": 27.873924255371094, + "margin_dpo/margin_std": 53.34053039550781, + "step": 168 + }, + { + "KL/chosen_KL_mean": -43.973472595214844, + "KL/mean": -56.64049530029297, + "KL/rejected_KL_mean": -69.30752563476562, + "KL/std": 43.73821258544922, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.33405113220215, + "fcm_dpo/q_t": 0.47141051292419434, + "grad_norm": 4.314192295074463, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.7225455641746521, + "logits/rejected": 0.6896216869354248, + "logps/chosen": -105.39933776855469, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -145.40342712402344, + "loss": 1.2896, + "margin_dpo/margin_mean": 25.33405113220215, + "margin_dpo/margin_std": 54.63517379760742, + "step": 169 + }, + { + "KL/chosen_KL_mean": -39.26036834716797, + "KL/mean": -53.16771697998047, + "KL/rejected_KL_mean": -67.07505798339844, + "KL/std": 41.21588134765625, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 27.814695358276367, + "fcm_dpo/q_t": 0.46871745586395264, + "grad_norm": 5.0231099128723145, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.689607560634613, + "logits/rejected": 0.6932332515716553, + "logps/chosen": -95.91355895996094, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -130.5347137451172, + "loss": 1.2806, + "margin_dpo/margin_mean": 27.814697265625, + "margin_dpo/margin_std": 54.60728454589844, + "step": 170 + }, + { + "KL/chosen_KL_mean": -52.41896438598633, + "KL/mean": -69.64653778076172, + "KL/rejected_KL_mean": -86.87411499023438, + "KL/std": 57.269317626953125, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.45514678955078, + "fcm_dpo/q_t": 0.46149614453315735, + "grad_norm": 4.3705902099609375, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.767911434173584, + "logits/rejected": 0.7232061624526978, + "logps/chosen": -116.15372467041016, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -165.37738037109375, + "loss": 1.2571, + "margin_dpo/margin_mean": 34.45514678955078, + "margin_dpo/margin_std": 64.5282974243164, + "step": 171 + }, + { + "KL/chosen_KL_mean": -55.21881866455078, + "KL/mean": -71.57858276367188, + "KL/rejected_KL_mean": -87.93833923339844, + "KL/std": 53.955963134765625, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.004572372883558273, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.719520568847656, + "fcm_dpo/q_t": 0.4635423421859741, + "grad_norm": 6.561391353607178, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.7488380670547485, + "logits/rejected": 0.686285674571991, + "logps/chosen": -107.42057800292969, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -170.79119873046875, + "loss": 1.259, + "margin_dpo/margin_mean": 32.719520568847656, + "margin_dpo/margin_std": 55.974552154541016, + "step": 172 + }, + { + "KL/chosen_KL_mean": -52.73722839355469, + "KL/mean": -72.90831756591797, + "KL/rejected_KL_mean": -93.07940673828125, + "KL/std": 59.073570251464844, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.014145536348223686, + "fcm_dpo/delta": 0.06264997273683548, + "fcm_dpo/margin": 40.342193603515625, + "fcm_dpo/q_t": 0.4352329969406128, + "grad_norm": 13.129067420959473, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.7639884352684021, + "logits/rejected": 0.6851507425308228, + "logps/chosen": -108.17195129394531, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -170.89907836914062, + "loss": 1.1037, + "margin_dpo/margin_mean": 40.342193603515625, + "margin_dpo/margin_std": 72.60908508300781, + "step": 173 + }, + { + "KL/chosen_KL_mean": -61.133522033691406, + "KL/mean": -83.47229766845703, + "KL/rejected_KL_mean": -105.81106567382812, + "KL/std": 60.41809844970703, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": -0.1022532507777214, + "fcm_dpo/margin": 44.67753601074219, + "fcm_dpo/q_t": 0.41912031173706055, + "grad_norm": 3.89009428024292, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.7750641107559204, + "logits/rejected": 0.7090386748313904, + "logps/chosen": -118.30548095703125, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -191.28684997558594, + "loss": 1.2858, + "margin_dpo/margin_mean": 44.67753601074219, + "margin_dpo/margin_std": 57.747344970703125, + "step": 174 + }, + { + "KL/chosen_KL_mean": -57.48314666748047, + "KL/mean": -79.11009216308594, + "KL/rejected_KL_mean": -100.73703002929688, + "KL/std": 61.092506408691406, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.253883361816406, + "fcm_dpo/q_t": 0.47405892610549927, + "grad_norm": 3.5935990810394287, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.799199104309082, + "logits/rejected": 0.7752283811569214, + "logps/chosen": -125.14877319335938, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -185.10470581054688, + "loss": 1.2931, + "margin_dpo/margin_mean": 43.253883361816406, + "margin_dpo/margin_std": 77.93309020996094, + "step": 175 + }, + { + "KL/chosen_KL_mean": -75.63528442382812, + "KL/mean": -89.86036682128906, + "KL/rejected_KL_mean": -104.08544158935547, + "KL/std": 67.660888671875, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.450157165527344, + "fcm_dpo/q_t": 0.4830489456653595, + "grad_norm": 4.446000576019287, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.7807189226150513, + "logits/rejected": 0.762683093547821, + "logps/chosen": -153.49404907226562, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -185.17276000976562, + "loss": 1.3297, + "margin_dpo/margin_mean": 28.450157165527344, + "margin_dpo/margin_std": 85.36820220947266, + "step": 176 + }, + { + "KL/chosen_KL_mean": -56.47824478149414, + "KL/mean": -86.89440155029297, + "KL/rejected_KL_mean": -117.310546875, + "KL/std": 66.56551361083984, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.83230209350586, + "fcm_dpo/q_t": 0.4634092152118683, + "grad_norm": 4.204408645629883, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.8780996799468994, + "logits/rejected": 0.7931272983551025, + "logps/chosen": -111.69863891601562, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -209.86029052734375, + "loss": 1.2499, + "margin_dpo/margin_mean": 60.832298278808594, + "margin_dpo/margin_std": 62.22540283203125, + "step": 177 + }, + { + "KL/chosen_KL_mean": -71.03286743164062, + "KL/mean": -90.96412658691406, + "KL/rejected_KL_mean": -110.89537048339844, + "KL/std": 68.37006378173828, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.86250305175781, + "fcm_dpo/q_t": 0.47601377964019775, + "grad_norm": 5.275820732116699, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.7893873453140259, + "logits/rejected": 0.7460411787033081, + "logps/chosen": -131.84335327148438, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -192.0251007080078, + "loss": 1.2979, + "margin_dpo/margin_mean": 39.86250686645508, + "margin_dpo/margin_std": 62.40123748779297, + "step": 178 + }, + { + "KL/chosen_KL_mean": -76.19467163085938, + "KL/mean": -97.18215942382812, + "KL/rejected_KL_mean": -118.16964721679688, + "KL/std": 71.81806945800781, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.9749755859375, + "fcm_dpo/q_t": 0.474868506193161, + "grad_norm": 3.9109606742858887, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.8630691170692444, + "logits/rejected": 0.8520238995552063, + "logps/chosen": -141.86639404296875, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -193.49551391601562, + "loss": 1.2974, + "margin_dpo/margin_mean": 41.9749755859375, + "margin_dpo/margin_std": 83.55865478515625, + "step": 179 + }, + { + "KL/chosen_KL_mean": -63.65786361694336, + "KL/mean": -79.41153717041016, + "KL/rejected_KL_mean": -95.16521453857422, + "KL/std": 63.367218017578125, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 31.507352828979492, + "fcm_dpo/q_t": 0.4810509979724884, + "grad_norm": 3.097667932510376, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.7191129922866821, + "logits/rejected": 0.7070032358169556, + "logps/chosen": -120.34066772460938, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -160.10935974121094, + "loss": 1.3188, + "margin_dpo/margin_mean": 31.507354736328125, + "margin_dpo/margin_std": 71.05530548095703, + "step": 180 + }, + { + "KL/chosen_KL_mean": -61.297054290771484, + "KL/mean": -88.07896423339844, + "KL/rejected_KL_mean": -114.86087036132812, + "KL/std": 77.27511596679688, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 53.56382751464844, + "fcm_dpo/q_t": 0.4678885340690613, + "grad_norm": 4.882580280303955, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.89243084192276, + "logits/rejected": 0.8556175827980042, + "logps/chosen": -122.07310485839844, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -198.844482421875, + "loss": 1.2695, + "margin_dpo/margin_mean": 53.56382369995117, + "margin_dpo/margin_std": 77.41310119628906, + "step": 181 + }, + { + "KL/chosen_KL_mean": -78.02749633789062, + "KL/mean": -106.99266815185547, + "KL/rejected_KL_mean": -135.95785522460938, + "KL/std": 78.79426574707031, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.93036651611328, + "fcm_dpo/q_t": 0.4655247628688812, + "grad_norm": 5.966950416564941, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.8725330829620361, + "logits/rejected": 0.7979344129562378, + "logps/chosen": -138.28128051757812, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -225.72848510742188, + "loss": 1.2631, + "margin_dpo/margin_mean": 57.93036651611328, + "margin_dpo/margin_std": 90.61637115478516, + "step": 182 + }, + { + "KL/chosen_KL_mean": -96.92032623291016, + "KL/mean": -109.78304290771484, + "KL/rejected_KL_mean": -122.645751953125, + "KL/std": 79.40963745117188, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.725440979003906, + "fcm_dpo/q_t": 0.48464155197143555, + "grad_norm": 4.779506683349609, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.8302876949310303, + "logits/rejected": 0.8160231113433838, + "logps/chosen": -158.68174743652344, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -195.19204711914062, + "loss": 1.3378, + "margin_dpo/margin_mean": 25.725440979003906, + "margin_dpo/margin_std": 93.7728271484375, + "step": 183 + }, + { + "KL/chosen_KL_mean": -62.56498718261719, + "KL/mean": -87.00849151611328, + "KL/rejected_KL_mean": -111.45199584960938, + "KL/std": 66.37240600585938, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.887001037597656, + "fcm_dpo/q_t": 0.4707995057106018, + "grad_norm": 6.9039998054504395, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.9358631372451782, + "logits/rejected": 0.8840564489364624, + "logps/chosen": -109.40570831298828, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -180.81292724609375, + "loss": 1.2804, + "margin_dpo/margin_mean": 48.887001037597656, + "margin_dpo/margin_std": 79.2603759765625, + "step": 184 + }, + { + "KL/chosen_KL_mean": -80.97999572753906, + "KL/mean": -106.63510131835938, + "KL/rejected_KL_mean": -132.29022216796875, + "KL/std": 85.46723937988281, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 51.310218811035156, + "fcm_dpo/q_t": 0.46929919719696045, + "grad_norm": 6.995913028717041, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.9254462122917175, + "logits/rejected": 0.8940056562423706, + "logps/chosen": -133.3011474609375, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -200.6787872314453, + "loss": 1.2813, + "margin_dpo/margin_mean": 51.31022262573242, + "margin_dpo/margin_std": 100.49153137207031, + "step": 185 + }, + { + "KL/chosen_KL_mean": -84.80559539794922, + "KL/mean": -113.83717346191406, + "KL/rejected_KL_mean": -142.86874389648438, + "KL/std": 107.6434555053711, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.063148498535156, + "fcm_dpo/q_t": 0.46672147512435913, + "grad_norm": 6.379158973693848, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.8682913184165955, + "logits/rejected": 0.8399237394332886, + "logps/chosen": -152.22572326660156, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -225.37843322753906, + "loss": 1.2795, + "margin_dpo/margin_mean": 58.063148498535156, + "margin_dpo/margin_std": 143.8511962890625, + "step": 186 + }, + { + "KL/chosen_KL_mean": -106.72709655761719, + "KL/mean": -132.92059326171875, + "KL/rejected_KL_mean": -159.11410522460938, + "KL/std": 99.10030364990234, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 52.38701248168945, + "fcm_dpo/q_t": 0.46877333521842957, + "grad_norm": 5.65391206741333, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.8852934241294861, + "logits/rejected": 0.8451642990112305, + "logps/chosen": -182.25259399414062, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -253.8769989013672, + "loss": 1.2815, + "margin_dpo/margin_mean": 52.38701629638672, + "margin_dpo/margin_std": 109.4050521850586, + "step": 187 + }, + { + "KL/chosen_KL_mean": -94.13689422607422, + "KL/mean": -118.1536865234375, + "KL/rejected_KL_mean": -142.17047119140625, + "KL/std": 93.5236587524414, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.0335807800293, + "fcm_dpo/q_t": 0.4715660512447357, + "grad_norm": 5.645846366882324, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.9208471775054932, + "logits/rejected": 0.8986366987228394, + "logps/chosen": -165.66021728515625, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -220.469970703125, + "loss": 1.2893, + "margin_dpo/margin_mean": 48.03358459472656, + "margin_dpo/margin_std": 105.69105529785156, + "step": 188 + }, + { + "KL/chosen_KL_mean": -113.9881591796875, + "KL/mean": -126.44110107421875, + "KL/rejected_KL_mean": -138.89404296875, + "KL/std": 106.01944732666016, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.0024263300001621246, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 24.9058895111084, + "fcm_dpo/q_t": 0.48495352268218994, + "grad_norm": 7.330818176269531, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.984262228012085, + "logits/rejected": 1.0013400316238403, + "logps/chosen": -186.1644287109375, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -214.15716552734375, + "loss": 1.3461, + "margin_dpo/margin_mean": 24.90589141845703, + "margin_dpo/margin_std": 109.00585174560547, + "step": 189 + }, + { + "KL/chosen_KL_mean": -87.60731506347656, + "KL/mean": -137.9809112548828, + "KL/rejected_KL_mean": -188.35450744628906, + "KL/std": 113.34170532226562, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.00413139583542943, + "fcm_dpo/delta": -0.01652386784553528, + "fcm_dpo/margin": 100.7471923828125, + "fcm_dpo/q_t": 0.4048711657524109, + "grad_norm": 11.910893440246582, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 1.0319868326187134, + "logits/rejected": 0.919386625289917, + "logps/chosen": -142.23158264160156, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -289.8251953125, + "loss": 1.0797, + "margin_dpo/margin_mean": 100.74720001220703, + "margin_dpo/margin_std": 116.15534973144531, + "step": 190 + }, + { + "KL/chosen_KL_mean": -128.3540802001953, + "KL/mean": -150.7650146484375, + "KL/rejected_KL_mean": -173.17596435546875, + "KL/std": 121.36798095703125, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.8218994140625, + "fcm_dpo/q_t": 0.4793294072151184, + "grad_norm": 7.475344181060791, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.9936637878417969, + "logits/rejected": 0.9684906005859375, + "logps/chosen": -201.28659057617188, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -263.12701416015625, + "loss": 1.3288, + "margin_dpo/margin_mean": 44.821895599365234, + "margin_dpo/margin_std": 163.15121459960938, + "step": 191 + }, + { + "KL/chosen_KL_mean": -115.16537475585938, + "KL/mean": -129.2557830810547, + "KL/rejected_KL_mean": -143.34620666503906, + "KL/std": 98.83819580078125, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.18083953857422, + "fcm_dpo/q_t": 0.48696666955947876, + "grad_norm": 3.313401222229004, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 1.02927827835083, + "logits/rejected": 1.007131576538086, + "logps/chosen": -169.16648864746094, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -206.8777618408203, + "loss": 1.3415, + "margin_dpo/margin_mean": 28.180843353271484, + "margin_dpo/margin_std": 89.10528564453125, + "step": 192 + }, + { + "KL/chosen_KL_mean": -104.46321105957031, + "KL/mean": -128.8253173828125, + "KL/rejected_KL_mean": -153.18743896484375, + "KL/std": 88.49467468261719, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.724239349365234, + "fcm_dpo/q_t": 0.47747933864593506, + "grad_norm": 3.7360472679138184, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 1.0135523080825806, + "logits/rejected": 1.0168956518173218, + "logps/chosen": -161.21249389648438, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -211.99374389648438, + "loss": 1.3057, + "margin_dpo/margin_mean": 48.7242431640625, + "margin_dpo/margin_std": 94.30784606933594, + "step": 193 + }, + { + "KL/chosen_KL_mean": -115.06211853027344, + "KL/mean": -141.86691284179688, + "KL/rejected_KL_mean": -168.67172241210938, + "KL/std": 117.1404800415039, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 53.60961151123047, + "fcm_dpo/q_t": 0.47572097182273865, + "grad_norm": 6.185281276702881, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 1.052534818649292, + "logits/rejected": 1.035210371017456, + "logps/chosen": -171.7115478515625, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -238.66127014160156, + "loss": 1.3049, + "margin_dpo/margin_mean": 53.60961151123047, + "margin_dpo/margin_std": 132.62509155273438, + "step": 194 + }, + { + "KL/chosen_KL_mean": -144.16168212890625, + "KL/mean": -165.70416259765625, + "KL/rejected_KL_mean": -187.2466278076172, + "KL/std": 127.5001220703125, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.0849494934082, + "fcm_dpo/q_t": 0.4802978038787842, + "grad_norm": 5.549439430236816, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.9337140321731567, + "logits/rejected": 0.9414522051811218, + "logps/chosen": -214.5714569091797, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -261.64111328125, + "loss": 1.326, + "margin_dpo/margin_mean": 43.08494567871094, + "margin_dpo/margin_std": 144.31869506835938, + "step": 195 + }, + { + "KL/chosen_KL_mean": -121.95760345458984, + "KL/mean": -155.3830108642578, + "KL/rejected_KL_mean": -188.80841064453125, + "KL/std": 146.87850952148438, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.8508071899414, + "fcm_dpo/q_t": 0.4697552025318146, + "grad_norm": 5.599655628204346, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 1.0035507678985596, + "logits/rejected": 0.9772269129753113, + "logps/chosen": -181.1851806640625, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -272.3559875488281, + "loss": 1.2815, + "margin_dpo/margin_mean": 66.8508071899414, + "margin_dpo/margin_std": 129.74188232421875, + "step": 196 + }, + { + "KL/chosen_KL_mean": -116.84405517578125, + "KL/mean": -151.54763793945312, + "KL/rejected_KL_mean": -186.251220703125, + "KL/std": 130.40406799316406, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.40716552734375, + "fcm_dpo/q_t": 0.4688175320625305, + "grad_norm": 4.895405292510986, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 1.0863635540008545, + "logits/rejected": 1.0468790531158447, + "logps/chosen": -168.37318420410156, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -259.95751953125, + "loss": 1.2843, + "margin_dpo/margin_mean": 69.40716552734375, + "margin_dpo/margin_std": 164.8914794921875, + "step": 197 + }, + { + "KL/chosen_KL_mean": -125.76316833496094, + "KL/mean": -156.66259765625, + "KL/rejected_KL_mean": -187.56204223632812, + "KL/std": 126.10319519042969, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.798851013183594, + "fcm_dpo/q_t": 0.4717727303504944, + "grad_norm": 4.650595664978027, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 1.1201140880584717, + "logits/rejected": 1.0980479717254639, + "logps/chosen": -185.54583740234375, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -259.807373046875, + "loss": 1.2896, + "margin_dpo/margin_mean": 61.79884719848633, + "margin_dpo/margin_std": 131.0926513671875, + "step": 198 + }, + { + "KL/chosen_KL_mean": -128.4625244140625, + "KL/mean": -160.48162841796875, + "KL/rejected_KL_mean": -192.500732421875, + "KL/std": 118.23385620117188, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.001862645847722888, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.0382308959961, + "fcm_dpo/q_t": 0.47144871950149536, + "grad_norm": 5.723056316375732, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 1.049532175064087, + "logits/rejected": 0.9984226226806641, + "logps/chosen": -184.84930419921875, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -267.06854248046875, + "loss": 1.2908, + "margin_dpo/margin_mean": 64.0382308959961, + "margin_dpo/margin_std": 147.04953002929688, + "step": 199 + }, + { + "KL/chosen_KL_mean": -128.42446899414062, + "KL/mean": -175.69747924804688, + "KL/rejected_KL_mean": -222.97048950195312, + "KL/std": 161.18472290039062, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.006587632466107607, + "fcm_dpo/delta": 0.0778110921382904, + "fcm_dpo/margin": 94.5460205078125, + "fcm_dpo/q_t": 0.4322563111782074, + "grad_norm": 19.645002365112305, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 1.102787971496582, + "logits/rejected": 1.0492348670959473, + "logps/chosen": -186.248779296875, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -312.2529296875, + "loss": 1.1653, + "margin_dpo/margin_mean": 94.5460205078125, + "margin_dpo/margin_std": 190.74359130859375, + "step": 200 + }, + { + "KL/chosen_KL_mean": -153.81317138671875, + "KL/mean": -185.98416137695312, + "KL/rejected_KL_mean": -218.15512084960938, + "KL/std": 170.94137573242188, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": -0.08959634602069855, + "fcm_dpo/margin": 64.3419418334961, + "fcm_dpo/q_t": 0.4359002113342285, + "grad_norm": 4.09077787399292, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 1.068925380706787, + "logits/rejected": 1.0387152433395386, + "logps/chosen": -212.8129425048828, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -302.83087158203125, + "loss": 1.3112, + "margin_dpo/margin_mean": 64.34194946289062, + "margin_dpo/margin_std": 189.60845947265625, + "step": 201 + }, + { + "KL/chosen_KL_mean": -113.20198822021484, + "KL/mean": -160.37185668945312, + "KL/rejected_KL_mean": -207.541748046875, + "KL/std": 120.42765808105469, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.33975982666016, + "fcm_dpo/q_t": 0.46428173780441284, + "grad_norm": 5.2453508377075195, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 1.0624161958694458, + "logits/rejected": 1.0226997137069702, + "logps/chosen": -160.86264038085938, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -281.17425537109375, + "loss": 1.2583, + "margin_dpo/margin_mean": 94.33975982666016, + "margin_dpo/margin_std": 137.9876708984375, + "step": 202 + }, + { + "KL/chosen_KL_mean": -150.9142303466797, + "KL/mean": -200.44468688964844, + "KL/rejected_KL_mean": -249.97515869140625, + "KL/std": 143.50985717773438, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.0609130859375, + "fcm_dpo/q_t": 0.4625791609287262, + "grad_norm": 6.239743232727051, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 1.1303417682647705, + "logits/rejected": 1.0701119899749756, + "logps/chosen": -213.23977661132812, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -349.347412109375, + "loss": 1.2536, + "margin_dpo/margin_mean": 99.0609130859375, + "margin_dpo/margin_std": 154.81906127929688, + "step": 203 + }, + { + "KL/chosen_KL_mean": -129.41946411132812, + "KL/mean": -163.46804809570312, + "KL/rejected_KL_mean": -197.5166015625, + "KL/std": 131.462158203125, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.09713745117188, + "fcm_dpo/q_t": 0.4740549921989441, + "grad_norm": 4.741433143615723, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 1.1037497520446777, + "logits/rejected": 1.0652885437011719, + "logps/chosen": -180.0487823486328, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -264.121337890625, + "loss": 1.3004, + "margin_dpo/margin_mean": 68.09713745117188, + "margin_dpo/margin_std": 162.0704803466797, + "step": 204 + }, + { + "KL/chosen_KL_mean": -178.677978515625, + "KL/mean": -224.63375854492188, + "KL/rejected_KL_mean": -270.58953857421875, + "KL/std": 211.23817443847656, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.91157531738281, + "fcm_dpo/q_t": 0.4660814702510834, + "grad_norm": 4.968754291534424, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 1.1142351627349854, + "logits/rejected": 1.0800881385803223, + "logps/chosen": -249.03414916992188, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -363.988037109375, + "loss": 1.2784, + "margin_dpo/margin_mean": 91.91158294677734, + "margin_dpo/margin_std": 221.230224609375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -190.87460327148438, + "KL/mean": -210.524169921875, + "KL/rejected_KL_mean": -230.17376708984375, + "KL/std": 179.7605743408203, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.29918670654297, + "fcm_dpo/q_t": 0.48608922958374023, + "grad_norm": 6.895484447479248, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 1.153550624847412, + "logits/rejected": 1.1396384239196777, + "logps/chosen": -258.52008056640625, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -310.0696105957031, + "loss": 1.3557, + "margin_dpo/margin_mean": 39.2991828918457, + "margin_dpo/margin_std": 221.7179412841797, + "step": 206 + }, + { + "KL/chosen_KL_mean": -192.2279052734375, + "KL/mean": -239.26425170898438, + "KL/rejected_KL_mean": -286.30059814453125, + "KL/std": 181.48193359375, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.07268524169922, + "fcm_dpo/q_t": 0.465423583984375, + "grad_norm": 4.847929954528809, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 1.124064326286316, + "logits/rejected": 1.0906260013580322, + "logps/chosen": -259.8921203613281, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -371.403076171875, + "loss": 1.2714, + "margin_dpo/margin_mean": 94.07269287109375, + "margin_dpo/margin_std": 202.92820739746094, + "step": 207 + }, + { + "KL/chosen_KL_mean": -176.33616638183594, + "KL/mean": -196.31283569335938, + "KL/rejected_KL_mean": -216.28952026367188, + "KL/std": 157.74716186523438, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.953346252441406, + "fcm_dpo/q_t": 0.48543232679367065, + "grad_norm": 5.455617427825928, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 1.1541500091552734, + "logits/rejected": 1.136106014251709, + "logps/chosen": -234.06787109375, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -290.4822998046875, + "loss": 1.3437, + "margin_dpo/margin_mean": 39.95335388183594, + "margin_dpo/margin_std": 167.1756134033203, + "step": 208 + }, + { + "KL/chosen_KL_mean": -220.13638305664062, + "KL/mean": -263.32781982421875, + "KL/rejected_KL_mean": -306.5192565917969, + "KL/std": 255.47915649414062, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.0015425452729687095, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.38284301757812, + "fcm_dpo/q_t": 0.46814075112342834, + "grad_norm": 6.359311580657959, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 1.1173896789550781, + "logits/rejected": 1.0917067527770996, + "logps/chosen": -288.68646240234375, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -394.4246826171875, + "loss": 1.3044, + "margin_dpo/margin_mean": 86.38285064697266, + "margin_dpo/margin_std": 283.98956298828125, + "step": 209 + }, + { + "KL/chosen_KL_mean": -179.193603515625, + "KL/mean": -246.36642456054688, + "KL/rejected_KL_mean": -313.53924560546875, + "KL/std": 254.68429565429688, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.0019031364936381578, + "fcm_dpo/delta": 0.023973697796463966, + "fcm_dpo/margin": 134.34561157226562, + "fcm_dpo/q_t": 0.45283159613609314, + "grad_norm": 9.294327735900879, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 1.2122018337249756, + "logits/rejected": 1.1607775688171387, + "logps/chosen": -236.46188354492188, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -399.2673034667969, + "loss": 1.198, + "margin_dpo/margin_mean": 134.34561157226562, + "margin_dpo/margin_std": 250.78106689453125, + "step": 210 + }, + { + "KL/chosen_KL_mean": -167.25848388671875, + "KL/mean": -236.9961395263672, + "KL/rejected_KL_mean": -306.7337951660156, + "KL/std": 225.39352416992188, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.004614786710590124, + "fcm_dpo/delta": -0.5421129465103149, + "fcm_dpo/margin": 139.4752960205078, + "fcm_dpo/q_t": 0.3901931047439575, + "grad_norm": 23.795278549194336, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 1.2343952655792236, + "logits/rejected": 1.1764979362487793, + "logps/chosen": -220.89920043945312, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -399.7725830078125, + "loss": 1.3759, + "margin_dpo/margin_mean": 139.4752960205078, + "margin_dpo/margin_std": 275.88897705078125, + "step": 211 + }, + { + "KL/chosen_KL_mean": -185.3031463623047, + "KL/mean": -240.04290771484375, + "KL/rejected_KL_mean": -294.78265380859375, + "KL/std": 222.0595703125, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.47950744628906, + "fcm_dpo/q_t": 0.49998939037323, + "grad_norm": 0.001861194847151637, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 1.2033497095108032, + "logits/rejected": 1.167579174041748, + "logps/chosen": -242.66989135742188, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -374.6790771484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.47950744628906, + "margin_dpo/margin_std": 288.2134704589844, + "step": 212 + }, + { + "KL/chosen_KL_mean": -211.92974853515625, + "KL/mean": -252.25604248046875, + "KL/rejected_KL_mean": -292.5823974609375, + "KL/std": 271.2462463378906, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.65261840820312, + "fcm_dpo/q_t": 0.4999921917915344, + "grad_norm": 0.0020023249089717865, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 1.1829383373260498, + "logits/rejected": 1.178153157234192, + "logps/chosen": -277.15087890625, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -372.7633972167969, + "loss": 1.3863, + "margin_dpo/margin_mean": 80.65261840820312, + "margin_dpo/margin_std": 359.29901123046875, + "step": 213 + }, + { + "KL/chosen_KL_mean": -217.9564208984375, + "KL/mean": -248.85366821289062, + "KL/rejected_KL_mean": -279.75091552734375, + "KL/std": 257.87176513671875, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.79451370239258, + "fcm_dpo/q_t": 0.49999403953552246, + "grad_norm": 0.0014851855812594295, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 1.1230930089950562, + "logits/rejected": 1.1280714273452759, + "logps/chosen": -279.2487487792969, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -347.4493408203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 61.794517517089844, + "margin_dpo/margin_std": 338.4102783203125, + "step": 214 + }, + { + "KL/chosen_KL_mean": -213.97157287597656, + "KL/mean": -296.07861328125, + "KL/rejected_KL_mean": -378.18560791015625, + "KL/std": 297.4847106933594, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.21408081054688, + "fcm_dpo/q_t": 0.49998411536216736, + "grad_norm": 0.003093123435974121, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 1.1404362916946411, + "logits/rejected": 1.096791386604309, + "logps/chosen": -277.8406982421875, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -476.95135498046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 164.21408081054688, + "margin_dpo/margin_std": 372.910400390625, + "step": 215 + }, + { + "KL/chosen_KL_mean": -207.86676025390625, + "KL/mean": -284.94354248046875, + "KL/rejected_KL_mean": -362.020263671875, + "KL/std": 262.0420837402344, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.15354919433594, + "fcm_dpo/q_t": 0.49998509883880615, + "grad_norm": 0.003247037995606661, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 1.1896278858184814, + "logits/rejected": 1.1537840366363525, + "logps/chosen": -275.69171142578125, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -458.422607421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 154.15353393554688, + "margin_dpo/margin_std": 316.47894287109375, + "step": 216 + }, + { + "KL/chosen_KL_mean": -208.4380645751953, + "KL/mean": -280.11431884765625, + "KL/rejected_KL_mean": -351.79052734375, + "KL/std": 250.4637908935547, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 143.35247802734375, + "fcm_dpo/q_t": 0.49998611211776733, + "grad_norm": 0.002652729395776987, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 1.1905633211135864, + "logits/rejected": 1.1464626789093018, + "logps/chosen": -268.94305419921875, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -436.0567321777344, + "loss": 1.3862, + "margin_dpo/margin_mean": 143.35247802734375, + "margin_dpo/margin_std": 322.68109130859375, + "step": 217 + }, + { + "KL/chosen_KL_mean": -261.1753845214844, + "KL/mean": -280.609130859375, + "KL/rejected_KL_mean": -300.0428466796875, + "KL/std": 274.44744873046875, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.86746597290039, + "fcm_dpo/q_t": 0.49999624490737915, + "grad_norm": 0.0014461593236774206, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 1.2026422023773193, + "logits/rejected": 1.20313560962677, + "logps/chosen": -331.76971435546875, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -373.9332275390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 38.867462158203125, + "margin_dpo/margin_std": 299.7601318359375, + "step": 218 + }, + { + "KL/chosen_KL_mean": -212.3046112060547, + "KL/mean": -256.71380615234375, + "KL/rejected_KL_mean": -301.1230163574219, + "KL/std": 195.33700561523438, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.81842041015625, + "fcm_dpo/q_t": 0.49999141693115234, + "grad_norm": 0.0015157065354287624, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 1.2033343315124512, + "logits/rejected": 1.186316967010498, + "logps/chosen": -272.7955322265625, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -376.9730224609375, + "loss": 1.3863, + "margin_dpo/margin_mean": 88.81842041015625, + "margin_dpo/margin_std": 246.12535095214844, + "step": 219 + }, + { + "KL/chosen_KL_mean": -152.8551025390625, + "KL/mean": -207.81396484375, + "KL/rejected_KL_mean": -262.7728271484375, + "KL/std": 192.8766326904297, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.91773986816406, + "fcm_dpo/q_t": 0.4999893307685852, + "grad_norm": 0.00206238916143775, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 1.2526534795761108, + "logits/rejected": 1.2216923236846924, + "logps/chosen": -197.86849975585938, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -333.26654052734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.91773986816406, + "margin_dpo/margin_std": 220.33956909179688, + "step": 220 + }, + { + "KL/chosen_KL_mean": -189.33047485351562, + "KL/mean": -266.7629089355469, + "KL/rejected_KL_mean": -344.19537353515625, + "KL/std": 269.7894287109375, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.86489868164062, + "fcm_dpo/q_t": 0.4999850392341614, + "grad_norm": 0.002508159726858139, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 1.1693766117095947, + "logits/rejected": 1.1354807615280151, + "logps/chosen": -248.4263153076172, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -432.8392333984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 154.86488342285156, + "margin_dpo/margin_std": 335.72869873046875, + "step": 221 + }, + { + "KL/chosen_KL_mean": -219.88815307617188, + "KL/mean": -301.087890625, + "KL/rejected_KL_mean": -382.28759765625, + "KL/std": 298.183349609375, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 162.39944458007812, + "fcm_dpo/q_t": 0.4999842941761017, + "grad_norm": 0.003092326922342181, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 1.2328643798828125, + "logits/rejected": 1.1719518899917603, + "logps/chosen": -275.8858337402344, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -494.2348937988281, + "loss": 1.3862, + "margin_dpo/margin_mean": 162.39944458007812, + "margin_dpo/margin_std": 360.7342529296875, + "step": 222 + }, + { + "KL/chosen_KL_mean": -218.1814422607422, + "KL/mean": -282.19677734375, + "KL/rejected_KL_mean": -346.21209716796875, + "KL/std": 276.27166748046875, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.03065490722656, + "fcm_dpo/q_t": 0.4999876022338867, + "grad_norm": 0.0024099198635667562, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 1.2278566360473633, + "logits/rejected": 1.2019100189208984, + "logps/chosen": -278.0728759765625, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -432.50164794921875, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.03065490722656, + "margin_dpo/margin_std": 309.4208679199219, + "step": 223 + }, + { + "KL/chosen_KL_mean": -233.63507080078125, + "KL/mean": -257.69244384765625, + "KL/rejected_KL_mean": -281.74981689453125, + "KL/std": 263.045166015625, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.114749908447266, + "fcm_dpo/q_t": 0.4999953508377075, + "grad_norm": 0.0018048178171738982, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 1.2750532627105713, + "logits/rejected": 1.2633410692214966, + "logps/chosen": -297.6797180175781, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -356.8043212890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 48.11475372314453, + "margin_dpo/margin_std": 270.31732177734375, + "step": 224 + }, + { + "KL/chosen_KL_mean": -257.35723876953125, + "KL/mean": -354.2248840332031, + "KL/rejected_KL_mean": -451.092529296875, + "KL/std": 388.58221435546875, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 193.7353057861328, + "fcm_dpo/q_t": 0.49998122453689575, + "grad_norm": 0.003719078376889229, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 1.2433876991271973, + "logits/rejected": 1.2075812816619873, + "logps/chosen": -323.45306396484375, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -548.779296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 193.73529052734375, + "margin_dpo/margin_std": 417.85693359375, + "step": 225 + }, + { + "KL/chosen_KL_mean": -184.22708129882812, + "KL/mean": -231.06289672851562, + "KL/rejected_KL_mean": -277.8987121582031, + "KL/std": 214.7318115234375, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.67162322998047, + "fcm_dpo/q_t": 0.49999094009399414, + "grad_norm": 0.0018737937789410353, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 1.2286086082458496, + "logits/rejected": 1.2162175178527832, + "logps/chosen": -235.64395141601562, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -344.19940185546875, + "loss": 1.3863, + "margin_dpo/margin_mean": 93.67161560058594, + "margin_dpo/margin_std": 288.7680358886719, + "step": 226 + }, + { + "KL/chosen_KL_mean": -203.26730346679688, + "KL/mean": -247.32821655273438, + "KL/rejected_KL_mean": -291.38916015625, + "KL/std": 236.57278442382812, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.12184143066406, + "fcm_dpo/q_t": 0.4999914765357971, + "grad_norm": 0.0017340828198939562, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 1.2588739395141602, + "logits/rejected": 1.2475314140319824, + "logps/chosen": -261.257080078125, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -366.44378662109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 88.12184143066406, + "margin_dpo/margin_std": 271.02618408203125, + "step": 227 + }, + { + "KL/chosen_KL_mean": -190.28192138671875, + "KL/mean": -252.29940795898438, + "KL/rejected_KL_mean": -314.31695556640625, + "KL/std": 251.63345336914062, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.0350112915039, + "fcm_dpo/q_t": 0.49998798966407776, + "grad_norm": 0.0025014770217239857, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 1.2646236419677734, + "logits/rejected": 1.2317761182785034, + "logps/chosen": -245.84127807617188, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -391.340576171875, + "loss": 1.3862, + "margin_dpo/margin_mean": 124.0350112915039, + "margin_dpo/margin_std": 312.650390625, + "step": 228 + }, + { + "KL/chosen_KL_mean": -173.30197143554688, + "KL/mean": -235.5060272216797, + "KL/rejected_KL_mean": -297.7100830078125, + "KL/std": 271.6685485839844, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.40812683105469, + "fcm_dpo/q_t": 0.49998795986175537, + "grad_norm": 0.0023492288310080767, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 1.2890279293060303, + "logits/rejected": 1.2542376518249512, + "logps/chosen": -224.09661865234375, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -376.157470703125, + "loss": 1.3862, + "margin_dpo/margin_mean": 124.40812683105469, + "margin_dpo/margin_std": 323.8258972167969, + "step": 229 + }, + { + "KL/chosen_KL_mean": -214.48695373535156, + "KL/mean": -247.63502502441406, + "KL/rejected_KL_mean": -280.7830810546875, + "KL/std": 275.4522705078125, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.2961196899414, + "fcm_dpo/q_t": 0.49999356269836426, + "grad_norm": 0.0015789554454386234, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 1.2542227506637573, + "logits/rejected": 1.2686011791229248, + "logps/chosen": -271.2161865234375, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -343.77490234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 66.2961196899414, + "margin_dpo/margin_std": 330.39410400390625, + "step": 230 + }, + { + "KL/chosen_KL_mean": -251.24246215820312, + "KL/mean": -291.47467041015625, + "KL/rejected_KL_mean": -331.7068786621094, + "KL/std": 262.79315185546875, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.46441650390625, + "fcm_dpo/q_t": 0.4999921917915344, + "grad_norm": 0.0015881173312664032, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 1.2051804065704346, + "logits/rejected": 1.2067768573760986, + "logps/chosen": -323.83953857421875, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -417.9391784667969, + "loss": 1.3863, + "margin_dpo/margin_mean": 80.46441650390625, + "margin_dpo/margin_std": 341.13519287109375, + "step": 231 + }, + { + "KL/chosen_KL_mean": -247.03854370117188, + "KL/mean": -286.765869140625, + "KL/rejected_KL_mean": -326.4931945800781, + "KL/std": 314.61639404296875, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.45465087890625, + "fcm_dpo/q_t": 0.499992311000824, + "grad_norm": 0.0016828159568831325, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 1.212486982345581, + "logits/rejected": 1.1974080801010132, + "logps/chosen": -315.1571044921875, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -410.287353515625, + "loss": 1.3863, + "margin_dpo/margin_mean": 79.45465087890625, + "margin_dpo/margin_std": 385.42352294921875, + "step": 232 + }, + { + "KL/chosen_KL_mean": -209.6459503173828, + "KL/mean": -239.39019775390625, + "KL/rejected_KL_mean": -269.1343994140625, + "KL/std": 206.01983642578125, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 59.48847198486328, + "fcm_dpo/q_t": 0.4999942183494568, + "grad_norm": 0.0019632186740636826, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 1.2710832357406616, + "logits/rejected": 1.2419123649597168, + "logps/chosen": -264.7160949707031, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -335.75286865234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 59.48847198486328, + "margin_dpo/margin_std": 254.86038208007812, + "step": 233 + }, + { + "KL/chosen_KL_mean": -177.97811889648438, + "KL/mean": -201.326416015625, + "KL/rejected_KL_mean": -224.67471313476562, + "KL/std": 178.39566040039062, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 46.69660568237305, + "fcm_dpo/q_t": 0.49999547004699707, + "grad_norm": 0.0013678089017048478, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 1.230893611907959, + "logits/rejected": 1.2400652170181274, + "logps/chosen": -233.9040069580078, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -275.7908020019531, + "loss": 1.3863, + "margin_dpo/margin_mean": 46.69660568237305, + "margin_dpo/margin_std": 229.29837036132812, + "step": 234 + }, + { + "KL/chosen_KL_mean": -215.87887573242188, + "KL/mean": -269.4683532714844, + "KL/rejected_KL_mean": -323.057861328125, + "KL/std": 284.41656494140625, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.17898559570312, + "fcm_dpo/q_t": 0.4999896287918091, + "grad_norm": 0.0015956625575199723, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 1.2453619241714478, + "logits/rejected": 1.2564499378204346, + "logps/chosen": -280.4186096191406, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -400.7493896484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.17898559570312, + "margin_dpo/margin_std": 307.4293212890625, + "step": 235 + }, + { + "KL/chosen_KL_mean": -268.6509704589844, + "KL/mean": -299.5308837890625, + "KL/rejected_KL_mean": -330.4107971191406, + "KL/std": 262.4644775390625, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.75979995727539, + "fcm_dpo/q_t": 0.4999940097332001, + "grad_norm": 0.0013398700393736362, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 1.2493544816970825, + "logits/rejected": 1.2446727752685547, + "logps/chosen": -339.80572509765625, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -415.29620361328125, + "loss": 1.3863, + "margin_dpo/margin_mean": 61.759796142578125, + "margin_dpo/margin_std": 344.2894287109375, + "step": 236 + }, + { + "KL/chosen_KL_mean": -239.0124969482422, + "KL/mean": -271.1570739746094, + "KL/rejected_KL_mean": -303.30169677734375, + "KL/std": 226.06419372558594, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.28919219970703, + "fcm_dpo/q_t": 0.49999377131462097, + "grad_norm": 0.0016555647598579526, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 1.203998327255249, + "logits/rejected": 1.211648941040039, + "logps/chosen": -315.1545104980469, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -384.1864929199219, + "loss": 1.3863, + "margin_dpo/margin_mean": 64.2891845703125, + "margin_dpo/margin_std": 271.71954345703125, + "step": 237 + }, + { + "KL/chosen_KL_mean": -209.5856475830078, + "KL/mean": -240.08590698242188, + "KL/rejected_KL_mean": -270.586181640625, + "KL/std": 247.57437133789062, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.00053405761719, + "fcm_dpo/q_t": 0.49999409914016724, + "grad_norm": 0.0015964685007929802, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 1.198110580444336, + "logits/rejected": 1.1851861476898193, + "logps/chosen": -278.4704895019531, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -346.48077392578125, + "loss": 1.3863, + "margin_dpo/margin_mean": 61.000526428222656, + "margin_dpo/margin_std": 281.09130859375, + "step": 238 + }, + { + "KL/chosen_KL_mean": -206.03028869628906, + "KL/mean": -302.91278076171875, + "KL/rejected_KL_mean": -399.7953186035156, + "KL/std": 299.86492919921875, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 193.76502990722656, + "fcm_dpo/q_t": 0.49998122453689575, + "grad_norm": 0.0037673686165362597, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 1.2611656188964844, + "logits/rejected": 1.1864782571792603, + "logps/chosen": -262.8021240234375, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -516.0258178710938, + "loss": 1.3862, + "margin_dpo/margin_mean": 193.76504516601562, + "margin_dpo/margin_std": 343.26318359375, + "step": 239 + }, + { + "KL/chosen_KL_mean": -220.20172119140625, + "KL/mean": -302.418212890625, + "KL/rejected_KL_mean": -384.63470458984375, + "KL/std": 321.66656494140625, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.43296813964844, + "fcm_dpo/q_t": 0.49998408555984497, + "grad_norm": 0.00302553316578269, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 1.2720229625701904, + "logits/rejected": 1.2404829263687134, + "logps/chosen": -273.55584716796875, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -464.7548828125, + "loss": 1.3862, + "margin_dpo/margin_mean": 164.43295288085938, + "margin_dpo/margin_std": 391.212890625, + "step": 240 + }, + { + "KL/chosen_KL_mean": -226.899658203125, + "KL/mean": -280.0133056640625, + "KL/rejected_KL_mean": -333.1269226074219, + "KL/std": 283.84722900390625, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.22727966308594, + "fcm_dpo/q_t": 0.49998971819877625, + "grad_norm": 0.0025036621373146772, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 1.2458418607711792, + "logits/rejected": 1.2394804954528809, + "logps/chosen": -298.7950744628906, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -416.161865234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 106.22727966308594, + "margin_dpo/margin_std": 347.1982421875, + "step": 241 + }, + { + "KL/chosen_KL_mean": -213.5513916015625, + "KL/mean": -256.9989318847656, + "KL/rejected_KL_mean": -300.44647216796875, + "KL/std": 220.97384643554688, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.89505004882812, + "fcm_dpo/q_t": 0.49999159574508667, + "grad_norm": 0.0013653092319145799, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 1.2530593872070312, + "logits/rejected": 1.252440094947815, + "logps/chosen": -271.47882080078125, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -368.28509521484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 86.89505767822266, + "margin_dpo/margin_std": 233.89271545410156, + "step": 242 + }, + { + "KL/chosen_KL_mean": -261.82867431640625, + "KL/mean": -280.63238525390625, + "KL/rejected_KL_mean": -299.43609619140625, + "KL/std": 240.62506103515625, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.60741424560547, + "fcm_dpo/q_t": 0.4999963641166687, + "grad_norm": 0.0014760139165446162, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 1.2306547164916992, + "logits/rejected": 1.242539405822754, + "logps/chosen": -336.1053466796875, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -372.67950439453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 37.60741424560547, + "margin_dpo/margin_std": 283.18731689453125, + "step": 243 + }, + { + "KL/chosen_KL_mean": -188.20855712890625, + "KL/mean": -253.76040649414062, + "KL/rejected_KL_mean": -319.312255859375, + "KL/std": 258.85791015625, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.1037139892578, + "fcm_dpo/q_t": 0.49998730421066284, + "grad_norm": 0.0026807633694261312, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 1.2482179403305054, + "logits/rejected": 1.2198402881622314, + "logps/chosen": -241.57244873046875, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -390.4150390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 131.1037139892578, + "margin_dpo/margin_std": 323.0280456542969, + "step": 244 + }, + { + "KL/chosen_KL_mean": -225.77105712890625, + "KL/mean": -294.74444580078125, + "KL/rejected_KL_mean": -363.7178649902344, + "KL/std": 293.7945556640625, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.94680786132812, + "fcm_dpo/q_t": 0.4999866485595703, + "grad_norm": 0.0021245733369141817, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 1.2210156917572021, + "logits/rejected": 1.2317429780960083, + "logps/chosen": -296.9661560058594, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -444.480224609375, + "loss": 1.3862, + "margin_dpo/margin_mean": 137.94680786132812, + "margin_dpo/margin_std": 270.6390075683594, + "step": 245 + }, + { + "KL/chosen_KL_mean": -266.89068603515625, + "KL/mean": -320.71197509765625, + "KL/rejected_KL_mean": -374.5332336425781, + "KL/std": 293.4222412109375, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.6425552368164, + "fcm_dpo/q_t": 0.4999895691871643, + "grad_norm": 0.0021127404179424047, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 1.265270471572876, + "logits/rejected": 1.237593650817871, + "logps/chosen": -338.5117492675781, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -468.567138671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.64256286621094, + "margin_dpo/margin_std": 309.1661376953125, + "step": 246 + }, + { + "KL/chosen_KL_mean": -302.8369140625, + "KL/mean": -326.8985595703125, + "KL/rejected_KL_mean": -350.9602355957031, + "KL/std": 334.1150817871094, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 48.12330627441406, + "fcm_dpo/q_t": 0.4999953508377075, + "grad_norm": 0.002026443835347891, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 1.2411327362060547, + "logits/rejected": 1.2402052879333496, + "logps/chosen": -382.85943603515625, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -440.1872863769531, + "loss": 1.3863, + "margin_dpo/margin_mean": 48.12330627441406, + "margin_dpo/margin_std": 404.8681640625, + "step": 247 + }, + { + "KL/chosen_KL_mean": -273.5013427734375, + "KL/mean": -274.2215881347656, + "KL/rejected_KL_mean": -274.94183349609375, + "KL/std": 300.5850830078125, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.44049072265625, + "fcm_dpo/q_t": 0.49999985098838806, + "grad_norm": 0.0020725768990814686, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 1.2465314865112305, + "logits/rejected": 1.2722697257995605, + "logps/chosen": -338.8793029785156, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -336.3076171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 1.44049072265625, + "margin_dpo/margin_std": 416.7322998046875, + "step": 248 + }, + { + "KL/chosen_KL_mean": -266.708984375, + "KL/mean": -261.5863037109375, + "KL/rejected_KL_mean": -256.463623046875, + "KL/std": 257.3307800292969, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -10.245379447937012, + "fcm_dpo/q_t": 0.5000010132789612, + "grad_norm": 0.0021689594723284245, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 1.2218635082244873, + "logits/rejected": 1.247019648551941, + "logps/chosen": -341.3104553222656, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -320.25701904296875, + "loss": 1.3863, + "margin_dpo/margin_mean": -10.245381355285645, + "margin_dpo/margin_std": 330.6340026855469, + "step": 249 + }, + { + "KL/chosen_KL_mean": -243.35177612304688, + "KL/mean": -302.5293884277344, + "KL/rejected_KL_mean": -361.70703125, + "KL/std": 364.4193115234375, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 118.3552474975586, + "fcm_dpo/q_t": 0.49998852610588074, + "grad_norm": 0.0022773679811507463, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 1.2806836366653442, + "logits/rejected": 1.28066885471344, + "logps/chosen": -305.28997802734375, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -433.9230651855469, + "loss": 1.3862, + "margin_dpo/margin_mean": 118.3552474975586, + "margin_dpo/margin_std": 424.7938537597656, + "step": 250 + }, + { + "KL/chosen_KL_mean": -238.873779296875, + "KL/mean": -284.80145263671875, + "KL/rejected_KL_mean": -330.7291259765625, + "KL/std": 258.41424560546875, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.85538482666016, + "fcm_dpo/q_t": 0.49999111890792847, + "grad_norm": 0.002100482815876603, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 1.240132451057434, + "logits/rejected": 1.2237201929092407, + "logps/chosen": -305.730712890625, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -415.5631103515625, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.85537719726562, + "margin_dpo/margin_std": 292.9080810546875, + "step": 251 + }, + { + "KL/chosen_KL_mean": -204.33717346191406, + "KL/mean": -248.474853515625, + "KL/rejected_KL_mean": -292.612548828125, + "KL/std": 251.9514923095703, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.27537536621094, + "fcm_dpo/q_t": 0.4999914765357971, + "grad_norm": 0.0017355838790535927, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 1.2478840351104736, + "logits/rejected": 1.2181034088134766, + "logps/chosen": -260.56109619140625, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -369.7261657714844, + "loss": 1.3863, + "margin_dpo/margin_mean": 88.27537536621094, + "margin_dpo/margin_std": 345.6888122558594, + "step": 252 + }, + { + "KL/chosen_KL_mean": -171.81988525390625, + "KL/mean": -219.31829833984375, + "KL/rejected_KL_mean": -266.81671142578125, + "KL/std": 209.09054565429688, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.99684143066406, + "fcm_dpo/q_t": 0.4999908208847046, + "grad_norm": 0.0018323827534914017, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 1.2669950723648071, + "logits/rejected": 1.26656174659729, + "logps/chosen": -224.02987670898438, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -325.5743408203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 94.99683380126953, + "margin_dpo/margin_std": 259.6316223144531, + "step": 253 + }, + { + "KL/chosen_KL_mean": -270.77874755859375, + "KL/mean": -316.91168212890625, + "KL/rejected_KL_mean": -363.0445556640625, + "KL/std": 343.09942626953125, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.26583862304688, + "fcm_dpo/q_t": 0.4999910593032837, + "grad_norm": 0.0022480462212115526, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 1.238560438156128, + "logits/rejected": 1.2194675207138062, + "logps/chosen": -336.4150695800781, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -445.3888244628906, + "loss": 1.3863, + "margin_dpo/margin_mean": 92.26583862304688, + "margin_dpo/margin_std": 337.8890380859375, + "step": 254 + }, + { + "KL/chosen_KL_mean": -255.44696044921875, + "KL/mean": -305.6675720214844, + "KL/rejected_KL_mean": -355.88818359375, + "KL/std": 318.76397705078125, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.44125366210938, + "fcm_dpo/q_t": 0.4999902844429016, + "grad_norm": 0.0018314458429813385, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 1.2495830059051514, + "logits/rejected": 1.2430639266967773, + "logps/chosen": -323.3580627441406, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -439.77935791015625, + "loss": 1.3863, + "margin_dpo/margin_mean": 100.44124603271484, + "margin_dpo/margin_std": 398.81390380859375, + "step": 255 + }, + { + "KL/chosen_KL_mean": -259.4943542480469, + "KL/mean": -310.67401123046875, + "KL/rejected_KL_mean": -361.85369873046875, + "KL/std": 321.3037414550781, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.35935974121094, + "fcm_dpo/q_t": 0.4999900758266449, + "grad_norm": 0.0024800151586532593, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 1.2634713649749756, + "logits/rejected": 1.2241191864013672, + "logps/chosen": -322.99432373046875, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -452.624755859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.35935974121094, + "margin_dpo/margin_std": 402.4932556152344, + "step": 256 + }, + { + "KL/chosen_KL_mean": -262.64410400390625, + "KL/mean": -331.921875, + "KL/rejected_KL_mean": -401.1995849609375, + "KL/std": 297.68072509765625, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.5554656982422, + "fcm_dpo/q_t": 0.49998658895492554, + "grad_norm": 0.0026133295614272356, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 1.2339143753051758, + "logits/rejected": 1.192386507987976, + "logps/chosen": -333.2447509765625, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -509.78271484375, + "loss": 1.3862, + "margin_dpo/margin_mean": 138.5554656982422, + "margin_dpo/margin_std": 377.41680908203125, + "step": 257 + }, + { + "KL/chosen_KL_mean": -226.23171997070312, + "KL/mean": -291.40777587890625, + "KL/rejected_KL_mean": -356.5838623046875, + "KL/std": 314.12420654296875, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.35211181640625, + "fcm_dpo/q_t": 0.49998739361763, + "grad_norm": 0.002391485497355461, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 1.226086139678955, + "logits/rejected": 1.204946517944336, + "logps/chosen": -285.48590087890625, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -442.1709289550781, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.35211181640625, + "margin_dpo/margin_std": 371.4399108886719, + "step": 258 + }, + { + "KL/chosen_KL_mean": -232.07115173339844, + "KL/mean": -321.46673583984375, + "KL/rejected_KL_mean": -410.86236572265625, + "KL/std": 316.74798583984375, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.79119873046875, + "fcm_dpo/q_t": 0.49998271465301514, + "grad_norm": 0.003150097792968154, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 1.2068517208099365, + "logits/rejected": 1.1704658269882202, + "logps/chosen": -297.50604248046875, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -506.2796630859375, + "loss": 1.3862, + "margin_dpo/margin_mean": 178.79119873046875, + "margin_dpo/margin_std": 385.9300537109375, + "step": 259 + }, + { + "KL/chosen_KL_mean": -184.13815307617188, + "KL/mean": -257.88519287109375, + "KL/rejected_KL_mean": -331.63226318359375, + "KL/std": 256.04803466796875, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 147.49411010742188, + "fcm_dpo/q_t": 0.4999857246875763, + "grad_norm": 0.002410466317087412, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 1.246681809425354, + "logits/rejected": 1.2082734107971191, + "logps/chosen": -233.2277374267578, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -410.64935302734375, + "loss": 1.3862, + "margin_dpo/margin_mean": 147.49411010742188, + "margin_dpo/margin_std": 310.56329345703125, + "step": 260 + }, + { + "KL/chosen_KL_mean": -234.46389770507812, + "KL/mean": -250.5965576171875, + "KL/rejected_KL_mean": -266.7292175292969, + "KL/std": 255.46902465820312, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 32.26532745361328, + "fcm_dpo/q_t": 0.4999969005584717, + "grad_norm": 0.001933022402226925, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 1.184006929397583, + "logits/rejected": 1.216488242149353, + "logps/chosen": -305.3363037109375, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -331.74444580078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 32.265323638916016, + "margin_dpo/margin_std": 333.6747741699219, + "step": 261 + }, + { + "KL/chosen_KL_mean": -257.26776123046875, + "KL/mean": -322.60302734375, + "KL/rejected_KL_mean": -387.93829345703125, + "KL/std": 322.8201904296875, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.67054748535156, + "fcm_dpo/q_t": 0.4999873638153076, + "grad_norm": 0.0025448398664593697, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 1.293022871017456, + "logits/rejected": 1.268181324005127, + "logps/chosen": -325.13836669921875, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -476.65887451171875, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.67056274414062, + "margin_dpo/margin_std": 383.709716796875, + "step": 262 + }, + { + "KL/chosen_KL_mean": -204.814208984375, + "KL/mean": -286.87152099609375, + "KL/rejected_KL_mean": -368.9288024902344, + "KL/std": 260.5648193359375, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.11460876464844, + "fcm_dpo/q_t": 0.49998411536216736, + "grad_norm": 0.0031319339759647846, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 1.2445664405822754, + "logits/rejected": 1.1973962783813477, + "logps/chosen": -260.0087890625, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -449.46929931640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 164.11460876464844, + "margin_dpo/margin_std": 305.0133056640625, + "step": 263 + }, + { + "KL/chosen_KL_mean": -304.02825927734375, + "KL/mean": -360.13616943359375, + "KL/rejected_KL_mean": -416.244140625, + "KL/std": 354.64105224609375, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.21583557128906, + "fcm_dpo/q_t": 0.4999891221523285, + "grad_norm": 0.0019485560478642583, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 1.213137149810791, + "logits/rejected": 1.212262511253357, + "logps/chosen": -387.198974609375, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -504.58038330078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 112.21582794189453, + "margin_dpo/margin_std": 419.0724792480469, + "step": 264 + }, + { + "KL/chosen_KL_mean": -178.7972412109375, + "KL/mean": -248.697021484375, + "KL/rejected_KL_mean": -318.5968017578125, + "KL/std": 211.929931640625, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.79954528808594, + "fcm_dpo/q_t": 0.499986469745636, + "grad_norm": 0.0025794145185500383, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 1.2378535270690918, + "logits/rejected": 1.2246911525726318, + "logps/chosen": -230.4600830078125, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -385.7688903808594, + "loss": 1.3862, + "margin_dpo/margin_mean": 139.79954528808594, + "margin_dpo/margin_std": 223.987548828125, + "step": 265 + }, + { + "KL/chosen_KL_mean": -226.0594024658203, + "KL/mean": -293.44244384765625, + "KL/rejected_KL_mean": -360.8254699707031, + "KL/std": 271.87078857421875, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.7660675048828, + "fcm_dpo/q_t": 0.4999869465827942, + "grad_norm": 0.0027317116037011147, + "learning_rate": 3.75e-07, + "logits/chosen": 1.2706271409988403, + "logits/rejected": 1.2448146343231201, + "logps/chosen": -283.5098876953125, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -438.4337158203125, + "loss": 1.3862, + "margin_dpo/margin_mean": 134.76608276367188, + "margin_dpo/margin_std": 255.4769744873047, + "step": 266 + }, + { + "KL/chosen_KL_mean": -187.67311096191406, + "KL/mean": -220.25643920898438, + "KL/rejected_KL_mean": -252.83978271484375, + "KL/std": 226.619140625, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 65.16666412353516, + "fcm_dpo/q_t": 0.4999936819076538, + "grad_norm": 0.0015339229721575975, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 1.2775146961212158, + "logits/rejected": 1.2753092050552368, + "logps/chosen": -242.70846557617188, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -318.93511962890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 65.16666412353516, + "margin_dpo/margin_std": 247.80099487304688, + "step": 267 + }, + { + "KL/chosen_KL_mean": -225.61331176757812, + "KL/mean": -269.3935546875, + "KL/rejected_KL_mean": -313.173828125, + "KL/std": 249.95361328125, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.56051635742188, + "fcm_dpo/q_t": 0.4999915361404419, + "grad_norm": 0.0014259773306548595, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 1.2123204469680786, + "logits/rejected": 1.2190499305725098, + "logps/chosen": -290.68505859375, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -384.59869384765625, + "loss": 1.3863, + "margin_dpo/margin_mean": 87.56051635742188, + "margin_dpo/margin_std": 257.95928955078125, + "step": 268 + }, + { + "KL/chosen_KL_mean": -245.9951171875, + "KL/mean": -291.13848876953125, + "KL/rejected_KL_mean": -336.2818603515625, + "KL/std": 273.3773498535156, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.28675079345703, + "fcm_dpo/q_t": 0.499991238117218, + "grad_norm": 0.002284642308950424, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 1.2732834815979004, + "logits/rejected": 1.2535003423690796, + "logps/chosen": -313.13134765625, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -418.83966064453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 90.28675842285156, + "margin_dpo/margin_std": 311.7049560546875, + "step": 269 + }, + { + "KL/chosen_KL_mean": -231.08285522460938, + "KL/mean": -305.00152587890625, + "KL/rejected_KL_mean": -378.920166015625, + "KL/std": 334.8675537109375, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 147.83731079101562, + "fcm_dpo/q_t": 0.4999856948852539, + "grad_norm": 0.0029855347238481045, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 1.265000343322754, + "logits/rejected": 1.234115481376648, + "logps/chosen": -297.77154541015625, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -464.0814208984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 147.83731079101562, + "margin_dpo/margin_std": 414.43560791015625, + "step": 270 + }, + { + "KL/chosen_KL_mean": -261.5472106933594, + "KL/mean": -319.3465576171875, + "KL/rejected_KL_mean": -377.14593505859375, + "KL/std": 378.90277099609375, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.59872436523438, + "fcm_dpo/q_t": 0.4999887943267822, + "grad_norm": 0.002353003015741706, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 1.3018203973770142, + "logits/rejected": 1.2758502960205078, + "logps/chosen": -333.95477294921875, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -469.20904541015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 115.5987319946289, + "margin_dpo/margin_std": 363.4158935546875, + "step": 271 + }, + { + "KL/chosen_KL_mean": -240.21646118164062, + "KL/mean": -269.6424255371094, + "KL/rejected_KL_mean": -299.06842041015625, + "KL/std": 277.5693359375, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.8519287109375, + "fcm_dpo/q_t": 0.49999427795410156, + "grad_norm": 0.001313323387876153, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 1.288849949836731, + "logits/rejected": 1.30269455909729, + "logps/chosen": -306.81787109375, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -366.8117980957031, + "loss": 1.3863, + "margin_dpo/margin_mean": 58.8519287109375, + "margin_dpo/margin_std": 330.32196044921875, + "step": 272 + }, + { + "KL/chosen_KL_mean": -218.98214721679688, + "KL/mean": -287.33990478515625, + "KL/rejected_KL_mean": -355.6976318359375, + "KL/std": 279.4718017578125, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 136.71548461914062, + "fcm_dpo/q_t": 0.49998676776885986, + "grad_norm": 0.0026562055572867393, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 1.2323360443115234, + "logits/rejected": 1.204160451889038, + "logps/chosen": -276.3370361328125, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -439.86932373046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 136.71546936035156, + "margin_dpo/margin_std": 341.8489074707031, + "step": 273 + }, + { + "KL/chosen_KL_mean": -227.13885498046875, + "KL/mean": -261.77813720703125, + "KL/rejected_KL_mean": -296.41741943359375, + "KL/std": 222.0265350341797, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.27857971191406, + "fcm_dpo/q_t": 0.4999932646751404, + "grad_norm": 0.001474517397582531, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 1.2174702882766724, + "logits/rejected": 1.2115811109542847, + "logps/chosen": -286.78033447265625, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -364.71087646484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 69.27857208251953, + "margin_dpo/margin_std": 245.24844360351562, + "step": 274 + }, + { + "KL/chosen_KL_mean": -202.14822387695312, + "KL/mean": -266.6926574707031, + "KL/rejected_KL_mean": -331.2370910644531, + "KL/std": 231.38563537597656, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 129.0888671875, + "fcm_dpo/q_t": 0.49998751282691956, + "grad_norm": 0.0028466566000133753, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 1.286808967590332, + "logits/rejected": 1.2461142539978027, + "logps/chosen": -255.4148712158203, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -405.07769775390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 129.0888671875, + "margin_dpo/margin_std": 244.29052734375, + "step": 275 + }, + { + "KL/chosen_KL_mean": -184.27493286132812, + "KL/mean": -225.28565979003906, + "KL/rejected_KL_mean": -266.2964172363281, + "KL/std": 173.41802978515625, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 82.021484375, + "fcm_dpo/q_t": 0.4999920725822449, + "grad_norm": 0.0015512220561504364, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 1.2420344352722168, + "logits/rejected": 1.2362403869628906, + "logps/chosen": -237.29571533203125, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -327.86322021484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 82.021484375, + "margin_dpo/margin_std": 235.60247802734375, + "step": 276 + }, + { + "KL/chosen_KL_mean": -272.57080078125, + "KL/mean": -275.81597900390625, + "KL/rejected_KL_mean": -279.0611572265625, + "KL/std": 280.1846618652344, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.490352630615234, + "fcm_dpo/q_t": 0.49999934434890747, + "grad_norm": 0.0013179422821849585, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 1.2742948532104492, + "logits/rejected": 1.291219711303711, + "logps/chosen": -344.0037841796875, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -346.71966552734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 6.490348815917969, + "margin_dpo/margin_std": 291.0250244140625, + "step": 277 + }, + { + "KL/chosen_KL_mean": -271.00958251953125, + "KL/mean": -318.64300537109375, + "KL/rejected_KL_mean": -366.27642822265625, + "KL/std": 313.33929443359375, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.26683807373047, + "fcm_dpo/q_t": 0.4999907612800598, + "grad_norm": 0.0023758604656904936, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 1.2502532005310059, + "logits/rejected": 1.2217738628387451, + "logps/chosen": -338.120361328125, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -455.02496337890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 95.26683044433594, + "margin_dpo/margin_std": 256.470947265625, + "step": 278 + }, + { + "KL/chosen_KL_mean": -200.28704833984375, + "KL/mean": -251.19207763671875, + "KL/rejected_KL_mean": -302.09710693359375, + "KL/std": 222.035400390625, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.81007385253906, + "fcm_dpo/q_t": 0.49999016523361206, + "grad_norm": 0.002032017568126321, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 1.242897868156433, + "logits/rejected": 1.225640058517456, + "logps/chosen": -254.78451538085938, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -372.5208435058594, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.8100814819336, + "margin_dpo/margin_std": 249.4945068359375, + "step": 279 + }, + { + "KL/chosen_KL_mean": -205.01568603515625, + "KL/mean": -279.7425842285156, + "KL/rejected_KL_mean": -354.469482421875, + "KL/std": 266.3010559082031, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 149.45379638671875, + "fcm_dpo/q_t": 0.4999855160713196, + "grad_norm": 0.002008727751672268, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 1.2439789772033691, + "logits/rejected": 1.2364301681518555, + "logps/chosen": -265.4485168457031, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -432.8599853515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 149.45379638671875, + "margin_dpo/margin_std": 288.43994140625, + "step": 280 + }, + { + "KL/chosen_KL_mean": -224.62643432617188, + "KL/mean": -239.99485778808594, + "KL/rejected_KL_mean": -255.36328125, + "KL/std": 230.7904052734375, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 30.736845016479492, + "fcm_dpo/q_t": 0.49999701976776123, + "grad_norm": 0.0013363017933443189, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 1.2372968196868896, + "logits/rejected": 1.2535717487335205, + "logps/chosen": -284.90850830078125, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -317.40338134765625, + "loss": 1.3863, + "margin_dpo/margin_mean": 30.736841201782227, + "margin_dpo/margin_std": 231.83021545410156, + "step": 281 + }, + { + "KL/chosen_KL_mean": -230.42965698242188, + "KL/mean": -267.6971130371094, + "KL/rejected_KL_mean": -304.96453857421875, + "KL/std": 253.47097778320312, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.53489685058594, + "fcm_dpo/q_t": 0.4999927878379822, + "grad_norm": 0.0015713232569396496, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 1.2360237836837769, + "logits/rejected": 1.222773790359497, + "logps/chosen": -291.0535888671875, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -373.6385498046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 74.5348892211914, + "margin_dpo/margin_std": 305.5966796875, + "step": 282 + }, + { + "KL/chosen_KL_mean": -207.01063537597656, + "KL/mean": -313.7869873046875, + "KL/rejected_KL_mean": -420.56329345703125, + "KL/std": 381.99566650390625, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 213.55267333984375, + "fcm_dpo/q_t": 0.4999793469905853, + "grad_norm": 0.004121567588299513, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 1.2334487438201904, + "logits/rejected": 1.195540189743042, + "logps/chosen": -274.65838623046875, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -520.5316772460938, + "loss": 1.3862, + "margin_dpo/margin_mean": 213.5526580810547, + "margin_dpo/margin_std": 493.8293151855469, + "step": 283 + }, + { + "KL/chosen_KL_mean": -213.08843994140625, + "KL/mean": -276.4241027832031, + "KL/rejected_KL_mean": -339.759765625, + "KL/std": 258.0605773925781, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.67131805419922, + "fcm_dpo/q_t": 0.49998775124549866, + "grad_norm": 0.0023277702275663614, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 1.2803890705108643, + "logits/rejected": 1.249000906944275, + "logps/chosen": -270.0558776855469, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -426.12213134765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.67131805419922, + "margin_dpo/margin_std": 305.05230712890625, + "step": 284 + }, + { + "KL/chosen_KL_mean": -243.08445739746094, + "KL/mean": -282.5973815917969, + "KL/rejected_KL_mean": -322.11029052734375, + "KL/std": 253.41421508789062, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 79.02581024169922, + "fcm_dpo/q_t": 0.49999234080314636, + "grad_norm": 0.0040868110954761505, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 1.2414788007736206, + "logits/rejected": 1.229378342628479, + "logps/chosen": -314.7405700683594, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -403.74859619140625, + "loss": 1.3863, + "margin_dpo/margin_mean": 79.02581787109375, + "margin_dpo/margin_std": 294.8158264160156, + "step": 285 + }, + { + "KL/chosen_KL_mean": -242.94229125976562, + "KL/mean": -322.85455322265625, + "KL/rejected_KL_mean": -402.766845703125, + "KL/std": 337.2320251464844, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 159.82455444335938, + "fcm_dpo/q_t": 0.4999845325946808, + "grad_norm": 0.0029409886337816715, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 1.2100770473480225, + "logits/rejected": 1.176485300064087, + "logps/chosen": -304.0218200683594, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -494.0481262207031, + "loss": 1.3862, + "margin_dpo/margin_mean": 159.82455444335938, + "margin_dpo/margin_std": 381.1865539550781, + "step": 286 + }, + { + "KL/chosen_KL_mean": -163.37152099609375, + "KL/mean": -214.27389526367188, + "KL/rejected_KL_mean": -265.1762390136719, + "KL/std": 199.1695556640625, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.80474090576172, + "fcm_dpo/q_t": 0.49999016523361206, + "grad_norm": 0.0017581077991053462, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 1.243018388748169, + "logits/rejected": 1.237166166305542, + "logps/chosen": -209.40731811523438, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -325.1291809082031, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.80474090576172, + "margin_dpo/margin_std": 249.35662841796875, + "step": 287 + }, + { + "KL/chosen_KL_mean": -247.86630249023438, + "KL/mean": -297.19573974609375, + "KL/rejected_KL_mean": -346.5251159667969, + "KL/std": 270.19976806640625, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.65884399414062, + "fcm_dpo/q_t": 0.49999046325683594, + "grad_norm": 0.002023276872932911, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 1.1788934469223022, + "logits/rejected": 1.1556057929992676, + "logps/chosen": -313.25714111328125, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -435.06121826171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.6588363647461, + "margin_dpo/margin_std": 299.93707275390625, + "step": 288 + }, + { + "KL/chosen_KL_mean": -218.3133087158203, + "KL/mean": -268.06744384765625, + "KL/rejected_KL_mean": -317.82159423828125, + "KL/std": 313.28009033203125, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.50828552246094, + "fcm_dpo/q_t": 0.4999903440475464, + "grad_norm": 0.00193495349958539, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 1.2431552410125732, + "logits/rejected": 1.243152141571045, + "logps/chosen": -272.90692138671875, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -385.0301513671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.50830078125, + "margin_dpo/margin_std": 335.50543212890625, + "step": 289 + }, + { + "KL/chosen_KL_mean": -221.37855529785156, + "KL/mean": -276.5778503417969, + "KL/rejected_KL_mean": -331.77716064453125, + "KL/std": 264.361083984375, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.39859771728516, + "fcm_dpo/q_t": 0.4999893307685852, + "grad_norm": 0.001686166855506599, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 1.2381784915924072, + "logits/rejected": 1.1985375881195068, + "logps/chosen": -282.76312255859375, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -423.7049255371094, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.39860534667969, + "margin_dpo/margin_std": 287.36102294921875, + "step": 290 + }, + { + "KL/chosen_KL_mean": -188.7603759765625, + "KL/mean": -280.0110168457031, + "KL/rejected_KL_mean": -371.2616882324219, + "KL/std": 305.65924072265625, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.5012969970703, + "fcm_dpo/q_t": 0.4999823570251465, + "grad_norm": 0.0030123190954327583, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 1.3322393894195557, + "logits/rejected": 1.3048069477081299, + "logps/chosen": -239.6234130859375, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -453.4703674316406, + "loss": 1.3862, + "margin_dpo/margin_mean": 182.5012969970703, + "margin_dpo/margin_std": 367.24835205078125, + "step": 291 + }, + { + "KL/chosen_KL_mean": -225.71701049804688, + "KL/mean": -268.5120849609375, + "KL/rejected_KL_mean": -311.30718994140625, + "KL/std": 296.009033203125, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.59013366699219, + "fcm_dpo/q_t": 0.4999917149543762, + "grad_norm": 0.0019401878817006946, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 1.244645118713379, + "logits/rejected": 1.2302597761154175, + "logps/chosen": -290.06591796875, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -384.1715087890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 85.59012603759766, + "margin_dpo/margin_std": 375.63006591796875, + "step": 292 + }, + { + "KL/chosen_KL_mean": -197.04302978515625, + "KL/mean": -276.2276306152344, + "KL/rejected_KL_mean": -355.4122314453125, + "KL/std": 286.4289855957031, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 158.36923217773438, + "fcm_dpo/q_t": 0.4999846816062927, + "grad_norm": 0.002683456754311919, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 1.3139989376068115, + "logits/rejected": 1.2797266244888306, + "logps/chosen": -251.91250610351562, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -437.2708740234375, + "loss": 1.3862, + "margin_dpo/margin_mean": 158.36923217773438, + "margin_dpo/margin_std": 348.2574157714844, + "step": 293 + }, + { + "KL/chosen_KL_mean": -204.25213623046875, + "KL/mean": -246.90603637695312, + "KL/rejected_KL_mean": -289.5599365234375, + "KL/std": 281.533447265625, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 85.30781555175781, + "fcm_dpo/q_t": 0.4999917447566986, + "grad_norm": 0.0014272573171183467, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 1.244971752166748, + "logits/rejected": 1.2297112941741943, + "logps/chosen": -260.92303466796875, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -359.88812255859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 85.30781555175781, + "margin_dpo/margin_std": 254.677978515625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -193.20266723632812, + "KL/mean": -270.8068542480469, + "KL/rejected_KL_mean": -348.4110107421875, + "KL/std": 270.56781005859375, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 155.20838928222656, + "fcm_dpo/q_t": 0.4999849796295166, + "grad_norm": 0.0031207390129566193, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 1.2604093551635742, + "logits/rejected": 1.2129838466644287, + "logps/chosen": -243.60354614257812, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -431.84625244140625, + "loss": 1.3862, + "margin_dpo/margin_mean": 155.20838928222656, + "margin_dpo/margin_std": 278.3273010253906, + "step": 295 + }, + { + "KL/chosen_KL_mean": -232.06796264648438, + "KL/mean": -298.0061340332031, + "KL/rejected_KL_mean": -363.9443359375, + "KL/std": 285.2636413574219, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.87635803222656, + "fcm_dpo/q_t": 0.49998724460601807, + "grad_norm": 0.0020598177798092365, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 1.1887896060943604, + "logits/rejected": 1.1723339557647705, + "logps/chosen": -301.21832275390625, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -453.5459899902344, + "loss": 1.3862, + "margin_dpo/margin_mean": 131.87635803222656, + "margin_dpo/margin_std": 339.91888427734375, + "step": 296 + }, + { + "KL/chosen_KL_mean": -209.5154266357422, + "KL/mean": -267.8277587890625, + "KL/rejected_KL_mean": -326.14007568359375, + "KL/std": 264.28759765625, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.6246566772461, + "fcm_dpo/q_t": 0.49998870491981506, + "grad_norm": 0.0021352798212319613, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 1.2609082460403442, + "logits/rejected": 1.251166820526123, + "logps/chosen": -267.53173828125, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -396.0978698730469, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.6246566772461, + "margin_dpo/margin_std": 218.6939239501953, + "step": 297 + }, + { + "KL/chosen_KL_mean": -233.57615661621094, + "KL/mean": -274.2930908203125, + "KL/rejected_KL_mean": -315.010009765625, + "KL/std": 299.4732360839844, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.43384552001953, + "fcm_dpo/q_t": 0.49999210238456726, + "grad_norm": 0.0023514467757195234, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 1.2886168956756592, + "logits/rejected": 1.2632899284362793, + "logps/chosen": -289.7454833984375, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -383.5605163574219, + "loss": 1.3863, + "margin_dpo/margin_mean": 81.43385314941406, + "margin_dpo/margin_std": 265.78594970703125, + "step": 298 + }, + { + "KL/chosen_KL_mean": -219.94754028320312, + "KL/mean": -259.0394287109375, + "KL/rejected_KL_mean": -298.13128662109375, + "KL/std": 257.43145751953125, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.1837387084961, + "fcm_dpo/q_t": 0.4999924302101135, + "grad_norm": 0.0015664704842492938, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 1.1981974840164185, + "logits/rejected": 1.1902220249176025, + "logps/chosen": -282.2653503417969, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -370.7315673828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 78.18374633789062, + "margin_dpo/margin_std": 312.6253967285156, + "step": 299 + }, + { + "KL/chosen_KL_mean": -220.5366973876953, + "KL/mean": -281.3580322265625, + "KL/rejected_KL_mean": -342.1794128417969, + "KL/std": 277.378173828125, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.64270782470703, + "fcm_dpo/q_t": 0.4999881982803345, + "grad_norm": 0.0024519728031009436, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 1.2705652713775635, + "logits/rejected": 1.2518296241760254, + "logps/chosen": -280.91827392578125, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -417.63385009765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.6427001953125, + "margin_dpo/margin_std": 316.0571594238281, + "step": 300 + }, + { + "KL/chosen_KL_mean": -221.11685180664062, + "KL/mean": -255.0898895263672, + "KL/rejected_KL_mean": -289.06292724609375, + "KL/std": 244.7019805908203, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.9460678100586, + "fcm_dpo/q_t": 0.4999934434890747, + "grad_norm": 0.0015637945616617799, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 1.2348217964172363, + "logits/rejected": 1.2190877199172974, + "logps/chosen": -273.9677429199219, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -359.03875732421875, + "loss": 1.3863, + "margin_dpo/margin_mean": 67.94606018066406, + "margin_dpo/margin_std": 319.27197265625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -285.2535400390625, + "KL/mean": -327.0502624511719, + "KL/rejected_KL_mean": -368.8470153808594, + "KL/std": 318.28948974609375, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.593505859375, + "fcm_dpo/q_t": 0.49999189376831055, + "grad_norm": 0.002295486629009247, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 1.2259061336517334, + "logits/rejected": 1.203969955444336, + "logps/chosen": -352.22003173828125, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -456.942138671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 83.593505859375, + "margin_dpo/margin_std": 367.9168395996094, + "step": 302 + }, + { + "KL/chosen_KL_mean": -222.12728881835938, + "KL/mean": -289.8243103027344, + "KL/rejected_KL_mean": -357.5213317871094, + "KL/std": 335.4458923339844, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 135.39398193359375, + "fcm_dpo/q_t": 0.4999868869781494, + "grad_norm": 0.0023559981491416693, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 1.1870148181915283, + "logits/rejected": 1.1635546684265137, + "logps/chosen": -284.24884033203125, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -447.8333740234375, + "loss": 1.3862, + "margin_dpo/margin_mean": 135.3939971923828, + "margin_dpo/margin_std": 403.8096923828125, + "step": 303 + }, + { + "KL/chosen_KL_mean": -235.18692016601562, + "KL/mean": -295.52618408203125, + "KL/rejected_KL_mean": -355.86541748046875, + "KL/std": 304.51690673828125, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 120.678466796875, + "fcm_dpo/q_t": 0.499988317489624, + "grad_norm": 0.0019933602306991816, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 1.2784829139709473, + "logits/rejected": 1.2663378715515137, + "logps/chosen": -295.88201904296875, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -434.1179504394531, + "loss": 1.3862, + "margin_dpo/margin_mean": 120.67847442626953, + "margin_dpo/margin_std": 319.9189453125, + "step": 304 + }, + { + "KL/chosen_KL_mean": -272.2154541015625, + "KL/mean": -285.18829345703125, + "KL/rejected_KL_mean": -298.16107177734375, + "KL/std": 275.7799072265625, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 25.945667266845703, + "fcm_dpo/q_t": 0.49999749660491943, + "grad_norm": 0.0017613671952858567, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 1.2120630741119385, + "logits/rejected": 1.2468100786209106, + "logps/chosen": -344.9145812988281, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -363.81781005859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 25.945667266845703, + "margin_dpo/margin_std": 326.3166198730469, + "step": 305 + }, + { + "KL/chosen_KL_mean": -198.40487670898438, + "KL/mean": -271.77508544921875, + "KL/rejected_KL_mean": -345.14532470703125, + "KL/std": 292.58148193359375, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 146.74044799804688, + "fcm_dpo/q_t": 0.49998578429222107, + "grad_norm": 0.002900703577324748, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 1.2891820669174194, + "logits/rejected": 1.249337077140808, + "logps/chosen": -252.37539672851562, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -416.1695556640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 146.74044799804688, + "margin_dpo/margin_std": 371.2115478515625, + "step": 306 + }, + { + "KL/chosen_KL_mean": -208.95254516601562, + "KL/mean": -243.92665100097656, + "KL/rejected_KL_mean": -278.9007568359375, + "KL/std": 252.12908935546875, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.94822692871094, + "fcm_dpo/q_t": 0.4999932646751404, + "grad_norm": 0.0016844801139086485, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 1.2810804843902588, + "logits/rejected": 1.2607386112213135, + "logps/chosen": -266.36566162109375, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -347.58087158203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 69.94822692871094, + "margin_dpo/margin_std": 359.750732421875, + "step": 307 + }, + { + "KL/chosen_KL_mean": -280.36407470703125, + "KL/mean": -300.9876708984375, + "KL/rejected_KL_mean": -321.61126708984375, + "KL/std": 320.69512939453125, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.247154235839844, + "fcm_dpo/q_t": 0.49999600648880005, + "grad_norm": 0.00144182366784662, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 1.2935481071472168, + "logits/rejected": 1.287482738494873, + "logps/chosen": -346.962890625, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -395.9483947753906, + "loss": 1.3863, + "margin_dpo/margin_mean": 41.247154235839844, + "margin_dpo/margin_std": 429.9683837890625, + "step": 308 + }, + { + "KL/chosen_KL_mean": -248.10739135742188, + "KL/mean": -301.85650634765625, + "KL/rejected_KL_mean": -355.60565185546875, + "KL/std": 268.0228271484375, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.498291015625, + "fcm_dpo/q_t": 0.4999895989894867, + "grad_norm": 0.002324699657037854, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 1.25961434841156, + "logits/rejected": 1.245445728302002, + "logps/chosen": -313.50213623046875, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -431.31494140625, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.49828338623047, + "margin_dpo/margin_std": 310.15728759765625, + "step": 309 + }, + { + "KL/chosen_KL_mean": -295.44293212890625, + "KL/mean": -333.5084228515625, + "KL/rejected_KL_mean": -371.5738830566406, + "KL/std": 341.388671875, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.13092041015625, + "fcm_dpo/q_t": 0.49999260902404785, + "grad_norm": 0.0016792939277365804, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 1.264609456062317, + "logits/rejected": 1.2663490772247314, + "logps/chosen": -370.1112060546875, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -452.142822265625, + "loss": 1.3863, + "margin_dpo/margin_mean": 76.13092803955078, + "margin_dpo/margin_std": 361.7975769042969, + "step": 310 + }, + { + "KL/chosen_KL_mean": -204.85330200195312, + "KL/mean": -294.4068603515625, + "KL/rejected_KL_mean": -383.9604187011719, + "KL/std": 327.627685546875, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 179.10711669921875, + "fcm_dpo/q_t": 0.49998265504837036, + "grad_norm": 0.0032888497225940228, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 1.2235231399536133, + "logits/rejected": 1.1665198802947998, + "logps/chosen": -264.5913391113281, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -477.5679931640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 179.10711669921875, + "margin_dpo/margin_std": 429.142822265625, + "step": 311 + }, + { + "KL/chosen_KL_mean": -197.45437622070312, + "KL/mean": -267.0279541015625, + "KL/rejected_KL_mean": -336.6015319824219, + "KL/std": 230.93252563476562, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.14715576171875, + "fcm_dpo/q_t": 0.49998652935028076, + "grad_norm": 0.0022931136190891266, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 1.2868523597717285, + "logits/rejected": 1.2605751752853394, + "logps/chosen": -251.27081298828125, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -405.25909423828125, + "loss": 1.3862, + "margin_dpo/margin_mean": 139.1471710205078, + "margin_dpo/margin_std": 282.67626953125, + "step": 312 + }, + { + "KL/chosen_KL_mean": -232.93560791015625, + "KL/mean": -266.92388916015625, + "KL/rejected_KL_mean": -300.91217041015625, + "KL/std": 304.7332763671875, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.97655487060547, + "fcm_dpo/q_t": 0.4999934136867523, + "grad_norm": 0.0016308584017679095, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 1.3045995235443115, + "logits/rejected": 1.302783489227295, + "logps/chosen": -292.8929748535156, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -370.2294921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 67.9765625, + "margin_dpo/margin_std": 373.98345947265625, + "step": 313 + }, + { + "KL/chosen_KL_mean": -240.1637420654297, + "KL/mean": -261.7298583984375, + "KL/rejected_KL_mean": -283.29595947265625, + "KL/std": 226.59564208984375, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.13225555419922, + "fcm_dpo/q_t": 0.4999958276748657, + "grad_norm": 0.0014463362749665976, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 1.2623672485351562, + "logits/rejected": 1.2793068885803223, + "logps/chosen": -310.431884765625, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -352.53570556640625, + "loss": 1.3863, + "margin_dpo/margin_mean": 43.13225555419922, + "margin_dpo/margin_std": 263.3436279296875, + "step": 314 + }, + { + "KL/chosen_KL_mean": -246.52500915527344, + "KL/mean": -288.8182373046875, + "KL/rejected_KL_mean": -331.1114196777344, + "KL/std": 295.01458740234375, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 84.58641052246094, + "fcm_dpo/q_t": 0.49999183416366577, + "grad_norm": 0.0019202815601602197, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 1.2083520889282227, + "logits/rejected": 1.2015604972839355, + "logps/chosen": -314.3197021484375, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -405.66290283203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 84.58641052246094, + "margin_dpo/margin_std": 364.86529541015625, + "step": 315 + }, + { + "KL/chosen_KL_mean": -200.84915161132812, + "KL/mean": -298.4619445800781, + "KL/rejected_KL_mean": -396.07470703125, + "KL/std": 297.4620666503906, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 195.2255859375, + "fcm_dpo/q_t": 0.4999811053276062, + "grad_norm": 0.0035327484365552664, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 1.211460828781128, + "logits/rejected": 1.1569902896881104, + "logps/chosen": -256.13763427734375, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -492.2319641113281, + "loss": 1.3862, + "margin_dpo/margin_mean": 195.22557067871094, + "margin_dpo/margin_std": 394.2180480957031, + "step": 316 + }, + { + "KL/chosen_KL_mean": -203.72149658203125, + "KL/mean": -248.2875518798828, + "KL/rejected_KL_mean": -292.8536376953125, + "KL/std": 228.49501037597656, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.13215637207031, + "fcm_dpo/q_t": 0.49999138712882996, + "grad_norm": 0.0016370030352845788, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 1.2447960376739502, + "logits/rejected": 1.2230143547058105, + "logps/chosen": -258.3028564453125, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -365.6259460449219, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.13215637207031, + "margin_dpo/margin_std": 290.63079833984375, + "step": 317 + }, + { + "KL/chosen_KL_mean": -209.48098754882812, + "KL/mean": -272.79022216796875, + "KL/rejected_KL_mean": -336.09942626953125, + "KL/std": 281.35205078125, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.61843872070312, + "fcm_dpo/q_t": 0.49998778104782104, + "grad_norm": 0.0026248276699334383, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 1.296776294708252, + "logits/rejected": 1.268967866897583, + "logps/chosen": -262.3692321777344, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -416.73931884765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.6184310913086, + "margin_dpo/margin_std": 386.39801025390625, + "step": 318 + }, + { + "KL/chosen_KL_mean": -246.42294311523438, + "KL/mean": -298.90435791015625, + "KL/rejected_KL_mean": -351.38580322265625, + "KL/std": 296.37994384765625, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.962890625, + "fcm_dpo/q_t": 0.4999898672103882, + "grad_norm": 0.0018176967278122902, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 1.2094240188598633, + "logits/rejected": 1.211363434791565, + "logps/chosen": -310.7862548828125, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -430.8587646484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.96288299560547, + "margin_dpo/margin_std": 335.21063232421875, + "step": 319 + }, + { + "KL/chosen_KL_mean": -198.46250915527344, + "KL/mean": -244.89617919921875, + "KL/rejected_KL_mean": -291.32989501953125, + "KL/std": 290.3658142089844, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.86738586425781, + "fcm_dpo/q_t": 0.4999910295009613, + "grad_norm": 0.0021061012521386147, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 1.236513614654541, + "logits/rejected": 1.227114200592041, + "logps/chosen": -248.02125549316406, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -362.5643310546875, + "loss": 1.3863, + "margin_dpo/margin_mean": 92.86739349365234, + "margin_dpo/margin_std": 356.69232177734375, + "step": 320 + }, + { + "KL/chosen_KL_mean": -178.01779174804688, + "KL/mean": -218.20223999023438, + "KL/rejected_KL_mean": -258.3866882324219, + "KL/std": 188.52374267578125, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 80.36886596679688, + "fcm_dpo/q_t": 0.4999922215938568, + "grad_norm": 0.0019223628332838416, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 1.2715182304382324, + "logits/rejected": 1.2804362773895264, + "logps/chosen": -230.1030731201172, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -313.9734191894531, + "loss": 1.3863, + "margin_dpo/margin_mean": 80.36886596679688, + "margin_dpo/margin_std": 233.33819580078125, + "step": 321 + }, + { + "KL/chosen_KL_mean": -205.60140991210938, + "KL/mean": -281.4060974121094, + "KL/rejected_KL_mean": -357.21075439453125, + "KL/std": 279.41717529296875, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 151.609375, + "fcm_dpo/q_t": 0.49998533725738525, + "grad_norm": 0.0024833802599459887, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 1.2712690830230713, + "logits/rejected": 1.2434958219528198, + "logps/chosen": -253.00552368164062, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -430.63677978515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 151.609375, + "margin_dpo/margin_std": 249.66696166992188, + "step": 322 + }, + { + "KL/chosen_KL_mean": -240.9448699951172, + "KL/mean": -307.22540283203125, + "KL/rejected_KL_mean": -373.50592041015625, + "KL/std": 300.21624755859375, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 132.5610809326172, + "fcm_dpo/q_t": 0.4999871551990509, + "grad_norm": 0.0025218951050192118, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 1.174757480621338, + "logits/rejected": 1.1535224914550781, + "logps/chosen": -310.951171875, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -460.47283935546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 132.5610809326172, + "margin_dpo/margin_std": 364.93914794921875, + "step": 323 + }, + { + "KL/chosen_KL_mean": -213.75881958007812, + "KL/mean": -267.80682373046875, + "KL/rejected_KL_mean": -321.8548583984375, + "KL/std": 297.6637268066406, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.09601593017578, + "fcm_dpo/q_t": 0.4999895393848419, + "grad_norm": 0.00212281895801425, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 1.2725311517715454, + "logits/rejected": 1.250030755996704, + "logps/chosen": -269.64764404296875, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -397.0857238769531, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.09602355957031, + "margin_dpo/margin_std": 297.931640625, + "step": 324 + }, + { + "KL/chosen_KL_mean": -228.80210876464844, + "KL/mean": -300.92083740234375, + "KL/rejected_KL_mean": -373.03955078125, + "KL/std": 327.6212158203125, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 144.23745727539062, + "fcm_dpo/q_t": 0.49998605251312256, + "grad_norm": 0.002541890600696206, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 1.2248497009277344, + "logits/rejected": 1.2180355787277222, + "logps/chosen": -292.9491271972656, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -452.95098876953125, + "loss": 1.3862, + "margin_dpo/margin_mean": 144.23745727539062, + "margin_dpo/margin_std": 414.3074645996094, + "step": 325 + }, + { + "KL/chosen_KL_mean": -288.5466003417969, + "KL/mean": -298.1192626953125, + "KL/rejected_KL_mean": -307.69195556640625, + "KL/std": 281.703125, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.14533233642578, + "fcm_dpo/q_t": 0.49999815225601196, + "grad_norm": 0.0015972210094332695, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 1.2351655960083008, + "logits/rejected": 1.2426598072052002, + "logps/chosen": -364.07794189453125, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -384.28179931640625, + "loss": 1.3863, + "margin_dpo/margin_mean": 19.14533233642578, + "margin_dpo/margin_std": 339.36480712890625, + "step": 326 + }, + { + "KL/chosen_KL_mean": -287.528564453125, + "KL/mean": -324.689453125, + "KL/rejected_KL_mean": -361.85040283203125, + "KL/std": 321.2103271484375, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.32182312011719, + "fcm_dpo/q_t": 0.49999281764030457, + "grad_norm": 0.0020750907715409994, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 1.1873157024383545, + "logits/rejected": 1.1866263151168823, + "logps/chosen": -356.86572265625, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -435.2279052734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 74.32182312011719, + "margin_dpo/margin_std": 364.2970275878906, + "step": 327 + }, + { + "KL/chosen_KL_mean": -234.11676025390625, + "KL/mean": -285.86553955078125, + "KL/rejected_KL_mean": -337.6143798828125, + "KL/std": 277.8924865722656, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.49760437011719, + "fcm_dpo/q_t": 0.49998998641967773, + "grad_norm": 0.0018194402800872922, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 1.1889562606811523, + "logits/rejected": 1.171844244003296, + "logps/chosen": -295.822998046875, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -421.3524475097656, + "loss": 1.3863, + "margin_dpo/margin_mean": 103.49760437011719, + "margin_dpo/margin_std": 310.90997314453125, + "step": 328 + }, + { + "KL/chosen_KL_mean": -241.38299560546875, + "KL/mean": -305.01055908203125, + "KL/rejected_KL_mean": -368.63812255859375, + "KL/std": 312.40496826171875, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 127.255126953125, + "fcm_dpo/q_t": 0.4999876618385315, + "grad_norm": 0.002292838878929615, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 1.2591251134872437, + "logits/rejected": 1.2390198707580566, + "logps/chosen": -305.88140869140625, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -452.29730224609375, + "loss": 1.3862, + "margin_dpo/margin_mean": 127.255126953125, + "margin_dpo/margin_std": 396.40557861328125, + "step": 329 + }, + { + "KL/chosen_KL_mean": -189.281494140625, + "KL/mean": -244.40090942382812, + "KL/rejected_KL_mean": -299.52032470703125, + "KL/std": 252.27133178710938, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.23881530761719, + "fcm_dpo/q_t": 0.4999893307685852, + "grad_norm": 0.0021174049470573664, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 1.2592849731445312, + "logits/rejected": 1.2317944765090942, + "logps/chosen": -244.08615112304688, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -374.83978271484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.23881530761719, + "margin_dpo/margin_std": 354.689697265625, + "step": 330 + }, + { + "KL/chosen_KL_mean": -210.95858764648438, + "KL/mean": -264.14520263671875, + "KL/rejected_KL_mean": -317.331787109375, + "KL/std": 248.91436767578125, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.37322235107422, + "fcm_dpo/q_t": 0.49998968839645386, + "grad_norm": 0.0013487640535458922, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 1.2729997634887695, + "logits/rejected": 1.273748755455017, + "logps/chosen": -270.201171875, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -387.2066345214844, + "loss": 1.3863, + "margin_dpo/margin_mean": 106.37321472167969, + "margin_dpo/margin_std": 286.5001220703125, + "step": 331 + }, + { + "KL/chosen_KL_mean": -226.46791076660156, + "KL/mean": -281.8634033203125, + "KL/rejected_KL_mean": -337.2588806152344, + "KL/std": 250.1396484375, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.7909927368164, + "fcm_dpo/q_t": 0.4999893009662628, + "grad_norm": 0.0019972885493189096, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 1.2130635976791382, + "logits/rejected": 1.2021818161010742, + "logps/chosen": -293.5776672363281, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -414.37725830078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.7909927368164, + "margin_dpo/margin_std": 291.0926208496094, + "step": 332 + }, + { + "KL/chosen_KL_mean": -233.87643432617188, + "KL/mean": -303.2012939453125, + "KL/rejected_KL_mean": -372.526123046875, + "KL/std": 304.5091247558594, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.64968872070312, + "fcm_dpo/q_t": 0.49998658895492554, + "grad_norm": 0.0025588450953364372, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 1.2254364490509033, + "logits/rejected": 1.1889737844467163, + "logps/chosen": -292.257568359375, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -457.55450439453125, + "loss": 1.3862, + "margin_dpo/margin_mean": 138.64968872070312, + "margin_dpo/margin_std": 301.95068359375, + "step": 333 + }, + { + "KL/chosen_KL_mean": -223.56558227539062, + "KL/mean": -282.3580322265625, + "KL/rejected_KL_mean": -341.15045166015625, + "KL/std": 278.36016845703125, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.58486938476562, + "fcm_dpo/q_t": 0.4999886155128479, + "grad_norm": 0.0021759173832833767, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 1.2243032455444336, + "logits/rejected": 1.1974499225616455, + "logps/chosen": -290.45758056640625, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -432.9873962402344, + "loss": 1.3862, + "margin_dpo/margin_mean": 117.58486938476562, + "margin_dpo/margin_std": 335.0116882324219, + "step": 334 + }, + { + "KL/chosen_KL_mean": -209.07431030273438, + "KL/mean": -258.6519775390625, + "KL/rejected_KL_mean": -308.2296142578125, + "KL/std": 264.89276123046875, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.15528869628906, + "fcm_dpo/q_t": 0.49999040365219116, + "grad_norm": 0.0017754482105374336, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 1.212792992591858, + "logits/rejected": 1.2027299404144287, + "logps/chosen": -270.5887756347656, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -383.9187927246094, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.15528869628906, + "margin_dpo/margin_std": 291.6207275390625, + "step": 335 + }, + { + "KL/chosen_KL_mean": -274.1320495605469, + "KL/mean": -322.80322265625, + "KL/rejected_KL_mean": -371.474365234375, + "KL/std": 263.66119384765625, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.34229278564453, + "fcm_dpo/q_t": 0.4999905824661255, + "grad_norm": 0.002034288365393877, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 1.2427122592926025, + "logits/rejected": 1.2363805770874023, + "logps/chosen": -342.98211669921875, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -464.4703674316406, + "loss": 1.3863, + "margin_dpo/margin_mean": 97.34230041503906, + "margin_dpo/margin_std": 293.6593017578125, + "step": 336 + }, + { + "KL/chosen_KL_mean": -289.3245849609375, + "KL/mean": -334.5281066894531, + "KL/rejected_KL_mean": -379.7316589355469, + "KL/std": 313.8486328125, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.40706634521484, + "fcm_dpo/q_t": 0.4999912679195404, + "grad_norm": 0.0020725426729768515, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 1.1902989149093628, + "logits/rejected": 1.1826756000518799, + "logps/chosen": -362.51239013671875, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -466.62286376953125, + "loss": 1.3863, + "margin_dpo/margin_mean": 90.40706634521484, + "margin_dpo/margin_std": 352.4085693359375, + "step": 337 + }, + { + "KL/chosen_KL_mean": -244.63876342773438, + "KL/mean": -279.78643798828125, + "KL/rejected_KL_mean": -314.93414306640625, + "KL/std": 228.41464233398438, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.29537963867188, + "fcm_dpo/q_t": 0.4999932050704956, + "grad_norm": 0.0021237812470644712, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 1.2304810285568237, + "logits/rejected": 1.2233942747116089, + "logps/chosen": -308.578369140625, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -390.27655029296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 70.29537963867188, + "margin_dpo/margin_std": 193.20977783203125, + "step": 338 + }, + { + "KL/chosen_KL_mean": -184.398681640625, + "KL/mean": -237.96548461914062, + "KL/rejected_KL_mean": -291.5323181152344, + "KL/std": 243.76315307617188, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.13363647460938, + "fcm_dpo/q_t": 0.4999896287918091, + "grad_norm": 0.002144779311493039, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 1.333993673324585, + "logits/rejected": 1.2993600368499756, + "logps/chosen": -229.94781494140625, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -358.5805358886719, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.13363647460938, + "margin_dpo/margin_std": 302.3402099609375, + "step": 339 + }, + { + "KL/chosen_KL_mean": -211.99609375, + "KL/mean": -246.68463134765625, + "KL/rejected_KL_mean": -281.3731994628906, + "KL/std": 209.9356689453125, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.37712860107422, + "fcm_dpo/q_t": 0.49999329447746277, + "grad_norm": 0.0015614746371284127, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 1.2425800561904907, + "logits/rejected": 1.241082787513733, + "logps/chosen": -266.0017395019531, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -342.6876220703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 69.37713623046875, + "margin_dpo/margin_std": 277.55560302734375, + "step": 340 + }, + { + "KL/chosen_KL_mean": -240.11708068847656, + "KL/mean": -302.804931640625, + "KL/rejected_KL_mean": -365.4927062988281, + "KL/std": 279.3319091796875, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 125.37564086914062, + "fcm_dpo/q_t": 0.4999878406524658, + "grad_norm": 0.002194821136072278, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 1.2894377708435059, + "logits/rejected": 1.27662992477417, + "logps/chosen": -303.5121765136719, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -441.70245361328125, + "loss": 1.3862, + "margin_dpo/margin_mean": 125.37564086914062, + "margin_dpo/margin_std": 277.63555908203125, + "step": 341 + }, + { + "KL/chosen_KL_mean": -215.72718811035156, + "KL/mean": -266.249755859375, + "KL/rejected_KL_mean": -316.7723388671875, + "KL/std": 273.38848876953125, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.045166015625, + "fcm_dpo/q_t": 0.49999022483825684, + "grad_norm": 0.0017220752779394388, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 1.278684139251709, + "logits/rejected": 1.2664589881896973, + "logps/chosen": -268.7749938964844, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -385.0577697753906, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.045166015625, + "margin_dpo/margin_std": 286.5677795410156, + "step": 342 + }, + { + "KL/chosen_KL_mean": -263.2760009765625, + "KL/mean": -300.4712829589844, + "KL/rejected_KL_mean": -337.6665344238281, + "KL/std": 306.28350830078125, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 74.39053344726562, + "fcm_dpo/q_t": 0.4999927878379822, + "grad_norm": 0.0017567041795700788, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 1.2100257873535156, + "logits/rejected": 1.210863471031189, + "logps/chosen": -333.85455322265625, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -422.4052734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 74.3905258178711, + "margin_dpo/margin_std": 372.44439697265625, + "step": 343 + }, + { + "KL/chosen_KL_mean": -235.8098907470703, + "KL/mean": -311.6300048828125, + "KL/rejected_KL_mean": -387.4500732421875, + "KL/std": 312.3414001464844, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 151.64019775390625, + "fcm_dpo/q_t": 0.49998533725738525, + "grad_norm": 0.002630658680573106, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 1.2740308046340942, + "logits/rejected": 1.2405366897583008, + "logps/chosen": -291.62091064453125, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -472.2264404296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 151.6402130126953, + "margin_dpo/margin_std": 380.27703857421875, + "step": 344 + }, + { + "KL/chosen_KL_mean": -196.2252655029297, + "KL/mean": -252.94921875, + "KL/rejected_KL_mean": -309.6731872558594, + "KL/std": 231.7432861328125, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.44795227050781, + "fcm_dpo/q_t": 0.49998903274536133, + "grad_norm": 0.0021876078099012375, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 1.2337894439697266, + "logits/rejected": 1.21903395652771, + "logps/chosen": -254.0113525390625, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -388.5916748046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 113.44795227050781, + "margin_dpo/margin_std": 229.27005004882812, + "step": 345 + }, + { + "KL/chosen_KL_mean": -210.19422912597656, + "KL/mean": -292.7418212890625, + "KL/rejected_KL_mean": -375.2893981933594, + "KL/std": 307.3424072265625, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 165.09515380859375, + "fcm_dpo/q_t": 0.4999840259552002, + "grad_norm": 0.002935102442279458, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 1.247345209121704, + "logits/rejected": 1.2206707000732422, + "logps/chosen": -266.4793701171875, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -466.44244384765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 165.09515380859375, + "margin_dpo/margin_std": 327.62249755859375, + "step": 346 + }, + { + "KL/chosen_KL_mean": -215.6938018798828, + "KL/mean": -268.2672424316406, + "KL/rejected_KL_mean": -320.8406677246094, + "KL/std": 256.3416442871094, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.14689636230469, + "fcm_dpo/q_t": 0.4999898076057434, + "grad_norm": 0.001989042619243264, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 1.3327654600143433, + "logits/rejected": 1.3226079940795898, + "logps/chosen": -269.193359375, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -393.3663330078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 105.14688873291016, + "margin_dpo/margin_std": 277.4522705078125, + "step": 347 + }, + { + "KL/chosen_KL_mean": -194.62255859375, + "KL/mean": -241.3387908935547, + "KL/rejected_KL_mean": -288.0550537109375, + "KL/std": 223.19094848632812, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.4324722290039, + "fcm_dpo/q_t": 0.49999094009399414, + "grad_norm": 0.0018931980011984706, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 1.2694389820098877, + "logits/rejected": 1.2611613273620605, + "logps/chosen": -245.409423828125, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -356.6923522949219, + "loss": 1.3863, + "margin_dpo/margin_mean": 93.43247985839844, + "margin_dpo/margin_std": 223.0704345703125, + "step": 348 + }, + { + "KL/chosen_KL_mean": -211.4373779296875, + "KL/mean": -280.389404296875, + "KL/rejected_KL_mean": -349.3414306640625, + "KL/std": 292.76397705078125, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.904052734375, + "fcm_dpo/q_t": 0.4999866485595703, + "grad_norm": 0.0022273960057646036, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 1.2621893882751465, + "logits/rejected": 1.2139443159103394, + "logps/chosen": -264.76239013671875, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -432.55377197265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 137.904052734375, + "margin_dpo/margin_std": 315.9725036621094, + "step": 349 + }, + { + "KL/chosen_KL_mean": -224.52761840820312, + "KL/mean": -307.943359375, + "KL/rejected_KL_mean": -391.3591003417969, + "KL/std": 310.1175537109375, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.83148193359375, + "fcm_dpo/q_t": 0.49998384714126587, + "grad_norm": 0.002880116691812873, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 1.258007526397705, + "logits/rejected": 1.247691035270691, + "logps/chosen": -286.15338134765625, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -478.995361328125, + "loss": 1.3862, + "margin_dpo/margin_mean": 166.8314971923828, + "margin_dpo/margin_std": 310.433837890625, + "step": 350 + }, + { + "KL/chosen_KL_mean": -202.2751007080078, + "KL/mean": -260.5731506347656, + "KL/rejected_KL_mean": -318.87115478515625, + "KL/std": 201.6959991455078, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.59608459472656, + "fcm_dpo/q_t": 0.49998873472213745, + "grad_norm": 0.0021962763275951147, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 1.2232160568237305, + "logits/rejected": 1.193381905555725, + "logps/chosen": -258.53143310546875, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -397.987060546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.59608459472656, + "margin_dpo/margin_std": 279.30438232421875, + "step": 351 + }, + { + "KL/chosen_KL_mean": -229.2626953125, + "KL/mean": -293.04437255859375, + "KL/rejected_KL_mean": -356.8260498046875, + "KL/std": 303.4278564453125, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 127.56336975097656, + "fcm_dpo/q_t": 0.4999876618385315, + "grad_norm": 0.002237893408164382, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 1.2531712055206299, + "logits/rejected": 1.2404941320419312, + "logps/chosen": -292.31463623046875, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -442.346435546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 127.5633773803711, + "margin_dpo/margin_std": 380.3623962402344, + "step": 352 + }, + { + "KL/chosen_KL_mean": -227.08648681640625, + "KL/mean": -265.6427307128906, + "KL/rejected_KL_mean": -304.198974609375, + "KL/std": 260.7290954589844, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.11247253417969, + "fcm_dpo/q_t": 0.4999925494194031, + "grad_norm": 0.0013403355842456222, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 1.2461869716644287, + "logits/rejected": 1.2634531259536743, + "logps/chosen": -296.0956726074219, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -376.85736083984375, + "loss": 1.3863, + "margin_dpo/margin_mean": 77.11247253417969, + "margin_dpo/margin_std": 259.31402587890625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -149.09295654296875, + "KL/mean": -222.05581665039062, + "KL/rejected_KL_mean": -295.0186767578125, + "KL/std": 179.0128936767578, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 145.92572021484375, + "fcm_dpo/q_t": 0.49998587369918823, + "grad_norm": 0.002911218209192157, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 1.290527105331421, + "logits/rejected": 1.233759880065918, + "logps/chosen": -188.8813018798828, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -364.5875244140625, + "loss": 1.3862, + "margin_dpo/margin_mean": 145.92572021484375, + "margin_dpo/margin_std": 210.240966796875, + "step": 354 + }, + { + "KL/chosen_KL_mean": -183.08291625976562, + "KL/mean": -279.84710693359375, + "KL/rejected_KL_mean": -376.611328125, + "KL/std": 313.0096740722656, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 193.52838134765625, + "fcm_dpo/q_t": 0.49998125433921814, + "grad_norm": 0.0035407356917858124, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 1.2896933555603027, + "logits/rejected": 1.2377452850341797, + "logps/chosen": -229.3383026123047, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -454.8136901855469, + "loss": 1.3862, + "margin_dpo/margin_mean": 193.5283966064453, + "margin_dpo/margin_std": 347.1357421875, + "step": 355 + }, + { + "KL/chosen_KL_mean": -173.40316772460938, + "KL/mean": -241.93011474609375, + "KL/rejected_KL_mean": -310.45709228515625, + "KL/std": 237.36767578125, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.0539093017578, + "fcm_dpo/q_t": 0.4999867379665375, + "grad_norm": 0.0023308955132961273, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 1.2501168251037598, + "logits/rejected": 1.2139482498168945, + "logps/chosen": -221.309326171875, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -384.7510681152344, + "loss": 1.3862, + "margin_dpo/margin_mean": 137.0539093017578, + "margin_dpo/margin_std": 294.431640625, + "step": 356 + }, + { + "KL/chosen_KL_mean": -230.25607299804688, + "KL/mean": -282.31524658203125, + "KL/rejected_KL_mean": -334.3744201660156, + "KL/std": 269.92694091796875, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.11831665039062, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.0023283795453608036, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 1.2416390180587769, + "logits/rejected": 1.2526752948760986, + "logps/chosen": -292.8910827636719, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -399.4884033203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.11831665039062, + "margin_dpo/margin_std": 328.5772705078125, + "step": 357 + }, + { + "KL/chosen_KL_mean": -257.3409423828125, + "KL/mean": -282.57147216796875, + "KL/rejected_KL_mean": -307.80206298828125, + "KL/std": 286.5031433105469, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 50.46110153198242, + "fcm_dpo/q_t": 0.4999951124191284, + "grad_norm": 0.001599806360900402, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 1.2319351434707642, + "logits/rejected": 1.2462717294692993, + "logps/chosen": -324.550537109375, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -377.1492004394531, + "loss": 1.3863, + "margin_dpo/margin_mean": 50.46110153198242, + "margin_dpo/margin_std": 362.09979248046875, + "step": 358 + }, + { + "KL/chosen_KL_mean": -271.7464599609375, + "KL/mean": -301.0337219238281, + "KL/rejected_KL_mean": -330.32098388671875, + "KL/std": 316.7093811035156, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 58.574562072753906, + "fcm_dpo/q_t": 0.49999430775642395, + "grad_norm": 0.001978958724066615, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 1.267923355102539, + "logits/rejected": 1.248403787612915, + "logps/chosen": -334.272216796875, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -406.9521484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 58.574562072753906, + "margin_dpo/margin_std": 401.56103515625, + "step": 359 + }, + { + "KL/chosen_KL_mean": -245.56210327148438, + "KL/mean": -321.9141540527344, + "KL/rejected_KL_mean": -398.26617431640625, + "KL/std": 338.3564147949219, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.7041015625, + "fcm_dpo/q_t": 0.4999852180480957, + "grad_norm": 0.0029005235992372036, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 1.2112646102905273, + "logits/rejected": 1.183484673500061, + "logps/chosen": -309.0498352050781, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -488.9552917480469, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.7041015625, + "margin_dpo/margin_std": 392.829345703125, + "step": 360 + }, + { + "KL/chosen_KL_mean": -215.9862060546875, + "KL/mean": -271.5434875488281, + "KL/rejected_KL_mean": -327.1007385253906, + "KL/std": 243.81903076171875, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 111.11454772949219, + "fcm_dpo/q_t": 0.49998927116394043, + "grad_norm": 0.0022065271623432636, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 1.2864508628845215, + "logits/rejected": 1.2714757919311523, + "logps/chosen": -273.9033508300781, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -399.49163818359375, + "loss": 1.3863, + "margin_dpo/margin_mean": 111.11454772949219, + "margin_dpo/margin_std": 294.0084228515625, + "step": 361 + }, + { + "KL/chosen_KL_mean": -237.84043884277344, + "KL/mean": -358.2422790527344, + "KL/rejected_KL_mean": -478.6441345214844, + "KL/std": 465.93792724609375, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 240.80369567871094, + "fcm_dpo/q_t": 0.4999766945838928, + "grad_norm": 0.004349971190094948, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 1.2654731273651123, + "logits/rejected": 1.2318336963653564, + "logps/chosen": -301.2839050292969, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -582.0993041992188, + "loss": 1.3862, + "margin_dpo/margin_mean": 240.8037109375, + "margin_dpo/margin_std": 607.7028198242188, + "step": 362 + }, + { + "KL/chosen_KL_mean": -184.51370239257812, + "KL/mean": -285.02001953125, + "KL/rejected_KL_mean": -385.5263366699219, + "KL/std": 271.80780029296875, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 201.0126495361328, + "fcm_dpo/q_t": 0.4999805688858032, + "grad_norm": 0.003892081556841731, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 1.2803268432617188, + "logits/rejected": 1.2195180654525757, + "logps/chosen": -233.16552734375, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -474.1853942871094, + "loss": 1.3862, + "margin_dpo/margin_mean": 201.0126495361328, + "margin_dpo/margin_std": 300.37652587890625, + "step": 363 + }, + { + "KL/chosen_KL_mean": -215.3506622314453, + "KL/mean": -287.34576416015625, + "KL/rejected_KL_mean": -359.3408508300781, + "KL/std": 300.54931640625, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 143.99021911621094, + "fcm_dpo/q_t": 0.49998608231544495, + "grad_norm": 0.0026705956552177668, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 1.2083289623260498, + "logits/rejected": 1.180536150932312, + "logps/chosen": -273.22174072265625, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -440.2958984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 143.99021911621094, + "margin_dpo/margin_std": 418.8135986328125, + "step": 364 + }, + { + "KL/chosen_KL_mean": -223.299072265625, + "KL/mean": -262.5470275878906, + "KL/rejected_KL_mean": -301.79498291015625, + "KL/std": 329.4648742675781, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 78.49591064453125, + "fcm_dpo/q_t": 0.49999240040779114, + "grad_norm": 0.001699400832876563, + "learning_rate": 2.5e-07, + "logits/chosen": 1.2776089906692505, + "logits/rejected": 1.2857856750488281, + "logps/chosen": -288.2412414550781, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -376.65496826171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 78.49592590332031, + "margin_dpo/margin_std": 294.7704772949219, + "step": 365 + }, + { + "KL/chosen_KL_mean": -215.96405029296875, + "KL/mean": -249.5860595703125, + "KL/rejected_KL_mean": -283.2080383300781, + "KL/std": 302.95806884765625, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.2439956665039, + "fcm_dpo/q_t": 0.4999935030937195, + "grad_norm": 0.0016670352779328823, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 1.2348253726959229, + "logits/rejected": 1.2384909391403198, + "logps/chosen": -271.1300354003906, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -348.46923828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 67.2439956665039, + "margin_dpo/margin_std": 308.7232666015625, + "step": 366 + }, + { + "KL/chosen_KL_mean": -227.30758666992188, + "KL/mean": -292.5162048339844, + "KL/rejected_KL_mean": -357.724853515625, + "KL/std": 294.52685546875, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.4172821044922, + "fcm_dpo/q_t": 0.4999873638153076, + "grad_norm": 0.0024976215790957212, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 1.2854868173599243, + "logits/rejected": 1.2474390268325806, + "logps/chosen": -283.31805419921875, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -435.03497314453125, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.4172821044922, + "margin_dpo/margin_std": 318.589111328125, + "step": 367 + }, + { + "KL/chosen_KL_mean": -271.27618408203125, + "KL/mean": -293.41290283203125, + "KL/rejected_KL_mean": -315.54962158203125, + "KL/std": 280.5533447265625, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.273414611816406, + "fcm_dpo/q_t": 0.4999956488609314, + "grad_norm": 0.0014242986217141151, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 1.193532109260559, + "logits/rejected": 1.199931025505066, + "logps/chosen": -346.10546875, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -391.66644287109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 44.2734260559082, + "margin_dpo/margin_std": 339.95147705078125, + "step": 368 + }, + { + "KL/chosen_KL_mean": -228.98980712890625, + "KL/mean": -280.967529296875, + "KL/rejected_KL_mean": -332.9452209472656, + "KL/std": 287.7408752441406, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.9554214477539, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.002357279881834984, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 1.2877874374389648, + "logits/rejected": 1.2739431858062744, + "logps/chosen": -287.3160400390625, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -413.8670654296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 103.9554214477539, + "margin_dpo/margin_std": 325.900634765625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -201.3767852783203, + "KL/mean": -259.1314392089844, + "KL/rejected_KL_mean": -316.8861083984375, + "KL/std": 232.7076873779297, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.50931549072266, + "fcm_dpo/q_t": 0.4999887943267822, + "grad_norm": 0.002263600006699562, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 1.2925589084625244, + "logits/rejected": 1.2621781826019287, + "logps/chosen": -254.260498046875, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -396.322998046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 115.50932312011719, + "margin_dpo/margin_std": 267.6164855957031, + "step": 370 + }, + { + "KL/chosen_KL_mean": -184.48826599121094, + "KL/mean": -229.01632690429688, + "KL/rejected_KL_mean": -273.54437255859375, + "KL/std": 192.02371215820312, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.05611419677734, + "fcm_dpo/q_t": 0.49999138712882996, + "grad_norm": 0.0017809628043323755, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 1.3020191192626953, + "logits/rejected": 1.2898714542388916, + "logps/chosen": -233.7124786376953, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -336.8928527832031, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.05611419677734, + "margin_dpo/margin_std": 218.2142333984375, + "step": 371 + }, + { + "KL/chosen_KL_mean": -195.20889282226562, + "KL/mean": -253.14254760742188, + "KL/rejected_KL_mean": -311.076171875, + "KL/std": 244.80682373046875, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.86727142333984, + "fcm_dpo/q_t": 0.49998876452445984, + "grad_norm": 0.002640601247549057, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 1.274523138999939, + "logits/rejected": 1.2315776348114014, + "logps/chosen": -247.47845458984375, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -384.0714111328125, + "loss": 1.3862, + "margin_dpo/margin_mean": 115.86727905273438, + "margin_dpo/margin_std": 293.59796142578125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -264.95257568359375, + "KL/mean": -320.5806884765625, + "KL/rejected_KL_mean": -376.20880126953125, + "KL/std": 349.17852783203125, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 111.25619506835938, + "fcm_dpo/q_t": 0.49998921155929565, + "grad_norm": 0.0023425230756402016, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 1.284815788269043, + "logits/rejected": 1.2553908824920654, + "logps/chosen": -326.0655822753906, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -452.4573059082031, + "loss": 1.3863, + "margin_dpo/margin_mean": 111.25619506835938, + "margin_dpo/margin_std": 393.34063720703125, + "step": 373 + }, + { + "KL/chosen_KL_mean": -264.5904541015625, + "KL/mean": -314.9049072265625, + "KL/rejected_KL_mean": -365.2193298339844, + "KL/std": 318.2697448730469, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.6288833618164, + "fcm_dpo/q_t": 0.4999902546405792, + "grad_norm": 0.0017044921405613422, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 1.231360912322998, + "logits/rejected": 1.247937798500061, + "logps/chosen": -337.2596435546875, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -442.0509033203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 100.6288833618164, + "margin_dpo/margin_std": 454.28070068359375, + "step": 374 + }, + { + "KL/chosen_KL_mean": -232.35386657714844, + "KL/mean": -285.4949951171875, + "KL/rejected_KL_mean": -338.63616943359375, + "KL/std": 343.39801025390625, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.28231811523438, + "fcm_dpo/q_t": 0.49998971819877625, + "grad_norm": 0.002389972796663642, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 1.2415052652359009, + "logits/rejected": 1.22267484664917, + "logps/chosen": -290.03717041015625, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -417.9771423339844, + "loss": 1.3863, + "margin_dpo/margin_mean": 106.28231811523438, + "margin_dpo/margin_std": 401.72052001953125, + "step": 375 + }, + { + "KL/chosen_KL_mean": -196.7454071044922, + "KL/mean": -273.0303039550781, + "KL/rejected_KL_mean": -349.315185546875, + "KL/std": 331.4114990234375, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.56979370117188, + "fcm_dpo/q_t": 0.4999852180480957, + "grad_norm": 0.0025670428294688463, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 1.2817575931549072, + "logits/rejected": 1.2636425495147705, + "logps/chosen": -248.41946411132812, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -425.0123291015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.56979370117188, + "margin_dpo/margin_std": 369.35845947265625, + "step": 376 + }, + { + "KL/chosen_KL_mean": -192.5317840576172, + "KL/mean": -222.54251098632812, + "KL/rejected_KL_mean": -252.55325317382812, + "KL/std": 172.49461364746094, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.02146911621094, + "fcm_dpo/q_t": 0.4999941885471344, + "grad_norm": 0.0016488871769979596, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 1.273218035697937, + "logits/rejected": 1.2637633085250854, + "logps/chosen": -238.7103271484375, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -310.30975341796875, + "loss": 1.3863, + "margin_dpo/margin_mean": 60.02146911621094, + "margin_dpo/margin_std": 183.21182250976562, + "step": 377 + }, + { + "KL/chosen_KL_mean": -234.7102813720703, + "KL/mean": -283.94964599609375, + "KL/rejected_KL_mean": -333.1890563964844, + "KL/std": 307.97674560546875, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.47879028320312, + "fcm_dpo/q_t": 0.49999046325683594, + "grad_norm": 0.0019810660742223263, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 1.2981789112091064, + "logits/rejected": 1.2841894626617432, + "logps/chosen": -293.92913818359375, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -404.437255859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.47879028320312, + "margin_dpo/margin_std": 306.84649658203125, + "step": 378 + }, + { + "KL/chosen_KL_mean": -273.62860107421875, + "KL/mean": -351.3236083984375, + "KL/rejected_KL_mean": -429.01861572265625, + "KL/std": 358.9712829589844, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 155.38998413085938, + "fcm_dpo/q_t": 0.4999849498271942, + "grad_norm": 0.003134517464786768, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 1.227163553237915, + "logits/rejected": 1.1967265605926514, + "logps/chosen": -349.9451904296875, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -533.2806396484375, + "loss": 1.3862, + "margin_dpo/margin_mean": 155.38998413085938, + "margin_dpo/margin_std": 411.5760192871094, + "step": 379 + }, + { + "KL/chosen_KL_mean": -208.5601806640625, + "KL/mean": -263.50933837890625, + "KL/rejected_KL_mean": -318.45849609375, + "KL/std": 250.38766479492188, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.89828491210938, + "fcm_dpo/q_t": 0.49998939037323, + "grad_norm": 0.0019253168720752, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 1.2732884883880615, + "logits/rejected": 1.262009620666504, + "logps/chosen": -269.8433532714844, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -390.847412109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.89830017089844, + "margin_dpo/margin_std": 294.832275390625, + "step": 380 + }, + { + "KL/chosen_KL_mean": -244.27020263671875, + "KL/mean": -272.90728759765625, + "KL/rejected_KL_mean": -301.54437255859375, + "KL/std": 301.01202392578125, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.27418518066406, + "fcm_dpo/q_t": 0.4999944567680359, + "grad_norm": 0.0016863668570294976, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 1.2436158657073975, + "logits/rejected": 1.240373134613037, + "logps/chosen": -302.48419189453125, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -362.3310546875, + "loss": 1.3863, + "margin_dpo/margin_mean": 57.27417755126953, + "margin_dpo/margin_std": 403.81927490234375, + "step": 381 + }, + { + "KL/chosen_KL_mean": -229.37368774414062, + "KL/mean": -292.00982666015625, + "KL/rejected_KL_mean": -354.6460266113281, + "KL/std": 275.9203796386719, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 125.2723388671875, + "fcm_dpo/q_t": 0.4999878704547882, + "grad_norm": 0.0023127899039536715, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 1.258483648300171, + "logits/rejected": 1.2473549842834473, + "logps/chosen": -291.1990051269531, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -437.6913146972656, + "loss": 1.3862, + "margin_dpo/margin_mean": 125.27233123779297, + "margin_dpo/margin_std": 331.326416015625, + "step": 382 + }, + { + "KL/chosen_KL_mean": -284.44842529296875, + "KL/mean": -303.908447265625, + "KL/rejected_KL_mean": -323.3684997558594, + "KL/std": 256.37823486328125, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 38.92008972167969, + "fcm_dpo/q_t": 0.49999624490737915, + "grad_norm": 0.002672591246664524, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 1.2364006042480469, + "logits/rejected": 1.2648537158966064, + "logps/chosen": -365.01165771484375, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -397.99774169921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 38.92008590698242, + "margin_dpo/margin_std": 323.5065612792969, + "step": 383 + }, + { + "KL/chosen_KL_mean": -215.20654296875, + "KL/mean": -270.7156677246094, + "KL/rejected_KL_mean": -326.22479248046875, + "KL/std": 235.81796264648438, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 111.01825714111328, + "fcm_dpo/q_t": 0.49998927116394043, + "grad_norm": 0.0019059223122894764, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 1.2277659177780151, + "logits/rejected": 1.224064588546753, + "logps/chosen": -280.68170166015625, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -405.8985595703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 111.01826477050781, + "margin_dpo/margin_std": 284.46221923828125, + "step": 384 + }, + { + "KL/chosen_KL_mean": -261.8004150390625, + "KL/mean": -326.974365234375, + "KL/rejected_KL_mean": -392.1483154296875, + "KL/std": 286.7763977050781, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.34793090820312, + "fcm_dpo/q_t": 0.49998739361763, + "grad_norm": 0.002203842392191291, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 1.246370792388916, + "logits/rejected": 1.2299015522003174, + "logps/chosen": -327.85693359375, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -478.82855224609375, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.34791564941406, + "margin_dpo/margin_std": 344.5843811035156, + "step": 385 + }, + { + "KL/chosen_KL_mean": -280.88348388671875, + "KL/mean": -343.2861328125, + "KL/rejected_KL_mean": -405.68878173828125, + "KL/std": 325.25909423828125, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.80535125732422, + "fcm_dpo/q_t": 0.4999879002571106, + "grad_norm": 0.0024226726964116096, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 1.2367877960205078, + "logits/rejected": 1.2165486812591553, + "logps/chosen": -356.50714111328125, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -498.3121032714844, + "loss": 1.3862, + "margin_dpo/margin_mean": 124.80534362792969, + "margin_dpo/margin_std": 393.1319580078125, + "step": 386 + }, + { + "KL/chosen_KL_mean": -202.7679443359375, + "KL/mean": -288.2266540527344, + "KL/rejected_KL_mean": -373.68536376953125, + "KL/std": 276.45501708984375, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.91744995117188, + "fcm_dpo/q_t": 0.4999834895133972, + "grad_norm": 0.0029783707577735186, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 1.2404074668884277, + "logits/rejected": 1.191530704498291, + "logps/chosen": -249.98965454101562, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -461.02349853515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 170.9174346923828, + "margin_dpo/margin_std": 304.63897705078125, + "step": 387 + }, + { + "KL/chosen_KL_mean": -288.9200439453125, + "KL/mean": -319.6807861328125, + "KL/rejected_KL_mean": -350.4415588378906, + "KL/std": 325.09112548828125, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.52149963378906, + "fcm_dpo/q_t": 0.49999403953552246, + "grad_norm": 0.00200459873303771, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 1.2456412315368652, + "logits/rejected": 1.248414158821106, + "logps/chosen": -363.49951171875, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -430.36712646484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 61.52150344848633, + "margin_dpo/margin_std": 357.44195556640625, + "step": 388 + }, + { + "KL/chosen_KL_mean": -228.92892456054688, + "KL/mean": -281.46588134765625, + "KL/rejected_KL_mean": -334.0028381347656, + "KL/std": 260.0592041015625, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.07390594482422, + "fcm_dpo/q_t": 0.4999898374080658, + "grad_norm": 0.0018365130526944995, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 1.256210446357727, + "logits/rejected": 1.2504541873931885, + "logps/chosen": -290.55328369140625, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -410.51263427734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 105.07391357421875, + "margin_dpo/margin_std": 320.30426025390625, + "step": 389 + }, + { + "KL/chosen_KL_mean": -165.7669677734375, + "KL/mean": -217.87681579589844, + "KL/rejected_KL_mean": -269.9866943359375, + "KL/std": 203.6674346923828, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.21971130371094, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.0020355647429823875, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 1.3178033828735352, + "logits/rejected": 1.2974865436553955, + "logps/chosen": -211.63882446289062, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -331.29266357421875, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.21971130371094, + "margin_dpo/margin_std": 242.00851440429688, + "step": 390 + }, + { + "KL/chosen_KL_mean": -229.9234619140625, + "KL/mean": -298.4404296875, + "KL/rejected_KL_mean": -366.95745849609375, + "KL/std": 266.13519287109375, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.0339813232422, + "fcm_dpo/q_t": 0.4999867081642151, + "grad_norm": 0.0023842838127166033, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 1.2694813013076782, + "logits/rejected": 1.245664119720459, + "logps/chosen": -288.1104736328125, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -450.59185791015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 137.0339813232422, + "margin_dpo/margin_std": 317.8896484375, + "step": 391 + }, + { + "KL/chosen_KL_mean": -249.9038848876953, + "KL/mean": -310.838623046875, + "KL/rejected_KL_mean": -371.7734069824219, + "KL/std": 360.3361511230469, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.86954498291016, + "fcm_dpo/q_t": 0.4999881982803345, + "grad_norm": 0.0024389002937823534, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 1.2220652103424072, + "logits/rejected": 1.2007293701171875, + "logps/chosen": -319.6484375, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -465.8321838378906, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.86953735351562, + "margin_dpo/margin_std": 398.31768798828125, + "step": 392 + }, + { + "KL/chosen_KL_mean": -212.96188354492188, + "KL/mean": -276.4071044921875, + "KL/rejected_KL_mean": -339.852294921875, + "KL/std": 308.44964599609375, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.89041137695312, + "fcm_dpo/q_t": 0.49998772144317627, + "grad_norm": 0.00197499874047935, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 1.2748873233795166, + "logits/rejected": 1.260786771774292, + "logps/chosen": -265.2967834472656, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -414.1903991699219, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.89041137695312, + "margin_dpo/margin_std": 273.0295104980469, + "step": 393 + }, + { + "KL/chosen_KL_mean": -218.35079956054688, + "KL/mean": -262.255126953125, + "KL/rejected_KL_mean": -306.1594543457031, + "KL/std": 245.12030029296875, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.80865478515625, + "fcm_dpo/q_t": 0.4999915063381195, + "grad_norm": 0.0014762998325750232, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 1.280198097229004, + "logits/rejected": 1.2725932598114014, + "logps/chosen": -279.02691650390625, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -377.52020263671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 87.80865478515625, + "margin_dpo/margin_std": 251.21902465820312, + "step": 394 + }, + { + "KL/chosen_KL_mean": -204.20767211914062, + "KL/mean": -277.1258544921875, + "KL/rejected_KL_mean": -350.0440673828125, + "KL/std": 298.6371765136719, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 145.8363800048828, + "fcm_dpo/q_t": 0.49998587369918823, + "grad_norm": 0.0028106472454965115, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 1.2618656158447266, + "logits/rejected": 1.2272990942001343, + "logps/chosen": -254.81199645996094, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -427.13134765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 145.8363800048828, + "margin_dpo/margin_std": 384.5468444824219, + "step": 395 + }, + { + "KL/chosen_KL_mean": -191.80307006835938, + "KL/mean": -259.073486328125, + "KL/rejected_KL_mean": -326.34393310546875, + "KL/std": 249.66404724121094, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.54083251953125, + "fcm_dpo/q_t": 0.49998700618743896, + "grad_norm": 0.002482072915881872, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 1.263901948928833, + "logits/rejected": 1.2265499830245972, + "logps/chosen": -243.16268920898438, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -406.2375183105469, + "loss": 1.3862, + "margin_dpo/margin_mean": 134.54083251953125, + "margin_dpo/margin_std": 234.10260009765625, + "step": 396 + }, + { + "KL/chosen_KL_mean": -259.5587463378906, + "KL/mean": -302.8362121582031, + "KL/rejected_KL_mean": -346.1136474609375, + "KL/std": 284.9183654785156, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.55490112304688, + "fcm_dpo/q_t": 0.49999162554740906, + "grad_norm": 0.0018756233621388674, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 1.194448709487915, + "logits/rejected": 1.1737489700317383, + "logps/chosen": -326.0149841308594, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -431.86102294921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 86.55490112304688, + "margin_dpo/margin_std": 322.10992431640625, + "step": 397 + }, + { + "KL/chosen_KL_mean": -200.04078674316406, + "KL/mean": -269.54071044921875, + "KL/rejected_KL_mean": -339.0406494140625, + "KL/std": 272.4809875488281, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.99984741210938, + "fcm_dpo/q_t": 0.49998655915260315, + "grad_norm": 0.0020650143269449472, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 1.2693488597869873, + "logits/rejected": 1.2469160556793213, + "logps/chosen": -249.2850341796875, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -414.2301025390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 138.99984741210938, + "margin_dpo/margin_std": 358.0387268066406, + "step": 398 + }, + { + "KL/chosen_KL_mean": -245.84356689453125, + "KL/mean": -352.4913330078125, + "KL/rejected_KL_mean": -459.1390380859375, + "KL/std": 331.2461242675781, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 213.29550170898438, + "fcm_dpo/q_t": 0.4999793469905853, + "grad_norm": 0.0037792252842336893, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 1.2296762466430664, + "logits/rejected": 1.1701855659484863, + "logps/chosen": -314.15032958984375, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -572.409912109375, + "loss": 1.3862, + "margin_dpo/margin_mean": 213.29551696777344, + "margin_dpo/margin_std": 435.9686584472656, + "step": 399 + }, + { + "KL/chosen_KL_mean": -270.5350646972656, + "KL/mean": -334.7283630371094, + "KL/rejected_KL_mean": -398.9216613769531, + "KL/std": 345.2818908691406, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.38658142089844, + "fcm_dpo/q_t": 0.49998757243156433, + "grad_norm": 0.002531483769416809, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 1.2615954875946045, + "logits/rejected": 1.237792730331421, + "logps/chosen": -342.16156005859375, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -489.9093017578125, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.3865966796875, + "margin_dpo/margin_std": 375.5792541503906, + "step": 400 + }, + { + "KL/chosen_KL_mean": -182.42166137695312, + "KL/mean": -258.8485107421875, + "KL/rejected_KL_mean": -335.275390625, + "KL/std": 285.74456787109375, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.85372924804688, + "fcm_dpo/q_t": 0.4999852180480957, + "grad_norm": 0.002612342592328787, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 1.272200584411621, + "logits/rejected": 1.2434661388397217, + "logps/chosen": -236.14662170410156, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -410.33843994140625, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.85372924804688, + "margin_dpo/margin_std": 328.9865417480469, + "step": 401 + }, + { + "KL/chosen_KL_mean": -211.4462890625, + "KL/mean": -261.5096435546875, + "KL/rejected_KL_mean": -311.572998046875, + "KL/std": 243.91207885742188, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.12675476074219, + "fcm_dpo/q_t": 0.4999902844429016, + "grad_norm": 0.00198004231788218, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 1.2314531803131104, + "logits/rejected": 1.2497401237487793, + "logps/chosen": -273.3201904296875, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -377.7249755859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 100.12675476074219, + "margin_dpo/margin_std": 330.08746337890625, + "step": 402 + }, + { + "KL/chosen_KL_mean": -185.16749572753906, + "KL/mean": -260.25628662109375, + "KL/rejected_KL_mean": -335.34503173828125, + "KL/std": 266.97735595703125, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 150.1775665283203, + "fcm_dpo/q_t": 0.4999854564666748, + "grad_norm": 0.002466583391651511, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 1.3430713415145874, + "logits/rejected": 1.302276611328125, + "logps/chosen": -236.48899841308594, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -421.8851623535156, + "loss": 1.3862, + "margin_dpo/margin_mean": 150.1775665283203, + "margin_dpo/margin_std": 270.43560791015625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -225.32257080078125, + "KL/mean": -306.13555908203125, + "KL/rejected_KL_mean": -386.9486083984375, + "KL/std": 334.2081604003906, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 161.62606811523438, + "fcm_dpo/q_t": 0.49998435378074646, + "grad_norm": 0.003134074853733182, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 1.26384699344635, + "logits/rejected": 1.219362735748291, + "logps/chosen": -287.58544921875, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -482.138916015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 161.6260528564453, + "margin_dpo/margin_std": 429.6391296386719, + "step": 404 + }, + { + "KL/chosen_KL_mean": -192.32791137695312, + "KL/mean": -249.77906799316406, + "KL/rejected_KL_mean": -307.230224609375, + "KL/std": 221.09945678710938, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.90231323242188, + "fcm_dpo/q_t": 0.499988853931427, + "grad_norm": 0.0018737346399575472, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 1.316763997077942, + "logits/rejected": 1.30680513381958, + "logps/chosen": -242.91226196289062, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -372.66180419921875, + "loss": 1.3862, + "margin_dpo/margin_mean": 114.90231323242188, + "margin_dpo/margin_std": 245.88070678710938, + "step": 405 + }, + { + "KL/chosen_KL_mean": -180.87625122070312, + "KL/mean": -281.7327880859375, + "KL/rejected_KL_mean": -382.58935546875, + "KL/std": 283.203369140625, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 201.71310424804688, + "fcm_dpo/q_t": 0.49998044967651367, + "grad_norm": 0.0038125943392515182, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 1.2964931726455688, + "logits/rejected": 1.2286221981048584, + "logps/chosen": -229.87185668945312, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -475.06707763671875, + "loss": 1.3862, + "margin_dpo/margin_mean": 201.71310424804688, + "margin_dpo/margin_std": 359.033447265625, + "step": 406 + }, + { + "KL/chosen_KL_mean": -312.56707763671875, + "KL/mean": -398.27764892578125, + "KL/rejected_KL_mean": -483.9881896972656, + "KL/std": 455.59527587890625, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 171.42111206054688, + "fcm_dpo/q_t": 0.49998340010643005, + "grad_norm": 0.00319691258482635, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 1.2000561952590942, + "logits/rejected": 1.1917808055877686, + "logps/chosen": -401.9676513671875, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -583.2759399414062, + "loss": 1.3862, + "margin_dpo/margin_mean": 171.42111206054688, + "margin_dpo/margin_std": 497.2770080566406, + "step": 407 + }, + { + "KL/chosen_KL_mean": -188.9539794921875, + "KL/mean": -266.2425842285156, + "KL/rejected_KL_mean": -343.53118896484375, + "KL/std": 264.1246337890625, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.5772247314453, + "fcm_dpo/q_t": 0.4999850392341614, + "grad_norm": 0.0026424252428114414, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 1.2529544830322266, + "logits/rejected": 1.2337000370025635, + "logps/chosen": -243.65789794921875, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -417.5177001953125, + "loss": 1.3862, + "margin_dpo/margin_mean": 154.5772247314453, + "margin_dpo/margin_std": 305.14788818359375, + "step": 408 + }, + { + "KL/chosen_KL_mean": -219.13699340820312, + "KL/mean": -239.82040405273438, + "KL/rejected_KL_mean": -260.5038146972656, + "KL/std": 184.755859375, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 41.36683654785156, + "fcm_dpo/q_t": 0.49999600648880005, + "grad_norm": 0.0013175446074455976, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 1.251279592514038, + "logits/rejected": 1.262905240058899, + "logps/chosen": -281.2552185058594, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -322.43731689453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 41.36684036254883, + "margin_dpo/margin_std": 230.91448974609375, + "step": 409 + }, + { + "KL/chosen_KL_mean": -235.9901580810547, + "KL/mean": -287.7513122558594, + "KL/rejected_KL_mean": -339.5124206542969, + "KL/std": 274.4139404296875, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.52224731445312, + "fcm_dpo/q_t": 0.49998998641967773, + "grad_norm": 0.0023037977516651154, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 1.2742689847946167, + "logits/rejected": 1.2695902585983276, + "logps/chosen": -297.7928466796875, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -416.1124267578125, + "loss": 1.3863, + "margin_dpo/margin_mean": 103.52224731445312, + "margin_dpo/margin_std": 323.41107177734375, + "step": 410 + }, + { + "KL/chosen_KL_mean": -204.3567352294922, + "KL/mean": -268.3748779296875, + "KL/rejected_KL_mean": -332.39306640625, + "KL/std": 259.58526611328125, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.03631591796875, + "fcm_dpo/q_t": 0.4999876022338867, + "grad_norm": 0.002165102167055011, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 1.2589843273162842, + "logits/rejected": 1.2471109628677368, + "logps/chosen": -258.8021240234375, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -406.9581298828125, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.03631591796875, + "margin_dpo/margin_std": 278.3114013671875, + "step": 411 + }, + { + "KL/chosen_KL_mean": -205.3777313232422, + "KL/mean": -259.4642028808594, + "KL/rejected_KL_mean": -313.55072021484375, + "KL/std": 227.35836791992188, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.17296600341797, + "fcm_dpo/q_t": 0.4999895393848419, + "grad_norm": 0.0022113437298685312, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 1.2673554420471191, + "logits/rejected": 1.2555253505706787, + "logps/chosen": -260.6258239746094, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -382.51690673828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.17295837402344, + "margin_dpo/margin_std": 254.52096557617188, + "step": 412 + }, + { + "KL/chosen_KL_mean": -264.1756286621094, + "KL/mean": -309.2933654785156, + "KL/rejected_KL_mean": -354.4111328125, + "KL/std": 302.36083984375, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.23551940917969, + "fcm_dpo/q_t": 0.4999912679195404, + "grad_norm": 0.0019386338535696268, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 1.1930580139160156, + "logits/rejected": 1.1896083354949951, + "logps/chosen": -332.8963623046875, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -433.176513671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 90.23551940917969, + "margin_dpo/margin_std": 384.3971862792969, + "step": 413 + }, + { + "KL/chosen_KL_mean": -200.8385009765625, + "KL/mean": -264.8590087890625, + "KL/rejected_KL_mean": -328.8795166015625, + "KL/std": 333.3802490234375, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.04103088378906, + "fcm_dpo/q_t": 0.4999876022338867, + "grad_norm": 0.0021054602693766356, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 1.2829506397247314, + "logits/rejected": 1.2689223289489746, + "logps/chosen": -254.97671508789062, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -403.53692626953125, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.04103088378906, + "margin_dpo/margin_std": 383.669189453125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -208.382568359375, + "KL/mean": -246.01028442382812, + "KL/rejected_KL_mean": -283.6380310058594, + "KL/std": 253.46176147460938, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.25543212890625, + "fcm_dpo/q_t": 0.4999927282333374, + "grad_norm": 0.00155459635425359, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 1.2656569480895996, + "logits/rejected": 1.2652559280395508, + "logps/chosen": -264.3011169433594, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -345.3857421875, + "loss": 1.3863, + "margin_dpo/margin_mean": 75.25544738769531, + "margin_dpo/margin_std": 311.67486572265625, + "step": 415 + }, + { + "KL/chosen_KL_mean": -220.16928100585938, + "KL/mean": -272.12646484375, + "KL/rejected_KL_mean": -324.0836181640625, + "KL/std": 320.10504150390625, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.91433715820312, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.0020540771074593067, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 1.2827048301696777, + "logits/rejected": 1.2663986682891846, + "logps/chosen": -274.8923645019531, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -393.25750732421875, + "loss": 1.3863, + "margin_dpo/margin_mean": 103.91433715820312, + "margin_dpo/margin_std": 338.7135925292969, + "step": 416 + }, + { + "KL/chosen_KL_mean": -224.02847290039062, + "KL/mean": -288.44708251953125, + "KL/rejected_KL_mean": -352.8656921386719, + "KL/std": 303.87835693359375, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.83721923828125, + "fcm_dpo/q_t": 0.49998751282691956, + "grad_norm": 0.002455303678289056, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 1.3073334693908691, + "logits/rejected": 1.2928953170776367, + "logps/chosen": -280.8197326660156, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -421.6448974609375, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.83721923828125, + "margin_dpo/margin_std": 371.76605224609375, + "step": 417 + }, + { + "KL/chosen_KL_mean": -284.6827087402344, + "KL/mean": -312.7228088378906, + "KL/rejected_KL_mean": -340.7629089355469, + "KL/std": 332.7872314453125, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 56.08018112182617, + "fcm_dpo/q_t": 0.49999457597732544, + "grad_norm": 0.001611909014172852, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 1.2963645458221436, + "logits/rejected": 1.3024256229400635, + "logps/chosen": -353.79071044921875, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -415.854248046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 56.080177307128906, + "margin_dpo/margin_std": 397.62890625, + "step": 418 + }, + { + "KL/chosen_KL_mean": -200.85401916503906, + "KL/mean": -274.2613830566406, + "KL/rejected_KL_mean": -347.66876220703125, + "KL/std": 276.50970458984375, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 146.81472778320312, + "fcm_dpo/q_t": 0.49998578429222107, + "grad_norm": 0.002377911237999797, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 1.2625758647918701, + "logits/rejected": 1.2619942426681519, + "logps/chosen": -259.0257873535156, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -419.33941650390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 146.81472778320312, + "margin_dpo/margin_std": 333.00238037109375, + "step": 419 + }, + { + "KL/chosen_KL_mean": -232.24765014648438, + "KL/mean": -268.002197265625, + "KL/rejected_KL_mean": -303.7567443847656, + "KL/std": 265.8175048828125, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.50907897949219, + "fcm_dpo/q_t": 0.49999308586120605, + "grad_norm": 0.001728436560370028, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 1.2875854969024658, + "logits/rejected": 1.2710034847259521, + "logps/chosen": -289.3011474609375, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -366.427734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 71.50907897949219, + "margin_dpo/margin_std": 340.4837951660156, + "step": 420 + }, + { + "KL/chosen_KL_mean": -230.90493774414062, + "KL/mean": -285.153076171875, + "KL/rejected_KL_mean": -339.4012145996094, + "KL/std": 287.30902099609375, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.49628448486328, + "fcm_dpo/q_t": 0.49998950958251953, + "grad_norm": 0.002344191772863269, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 1.237176775932312, + "logits/rejected": 1.2174639701843262, + "logps/chosen": -288.22821044921875, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -414.7390441894531, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.49629211425781, + "margin_dpo/margin_std": 395.6588439941406, + "step": 421 + }, + { + "KL/chosen_KL_mean": -228.75091552734375, + "KL/mean": -281.6344909667969, + "KL/rejected_KL_mean": -334.51806640625, + "KL/std": 274.93902587890625, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 105.76715087890625, + "fcm_dpo/q_t": 0.49998974800109863, + "grad_norm": 0.0021835805382579565, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 1.231833577156067, + "logits/rejected": 1.2441257238388062, + "logps/chosen": -295.8084716796875, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -406.6460876464844, + "loss": 1.3863, + "margin_dpo/margin_mean": 105.76715087890625, + "margin_dpo/margin_std": 315.6912841796875, + "step": 422 + }, + { + "KL/chosen_KL_mean": -189.46343994140625, + "KL/mean": -263.8796691894531, + "KL/rejected_KL_mean": -338.2958679199219, + "KL/std": 262.6513671875, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.83242797851562, + "fcm_dpo/q_t": 0.49998557567596436, + "grad_norm": 0.0025841707829385996, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 1.2950785160064697, + "logits/rejected": 1.2744381427764893, + "logps/chosen": -243.52511596679688, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -414.9367980957031, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.83242797851562, + "margin_dpo/margin_std": 266.20477294921875, + "step": 423 + }, + { + "KL/chosen_KL_mean": -204.17025756835938, + "KL/mean": -266.5606384277344, + "KL/rejected_KL_mean": -328.95098876953125, + "KL/std": 212.84597778320312, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.78072357177734, + "fcm_dpo/q_t": 0.499987930059433, + "grad_norm": 0.002034224336966872, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 1.2100038528442383, + "logits/rejected": 1.197531819343567, + "logps/chosen": -257.7791442871094, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -408.16497802734375, + "loss": 1.3862, + "margin_dpo/margin_mean": 124.78071594238281, + "margin_dpo/margin_std": 222.72096252441406, + "step": 424 + }, + { + "KL/chosen_KL_mean": -202.778564453125, + "KL/mean": -247.59307861328125, + "KL/rejected_KL_mean": -292.4075927734375, + "KL/std": 240.92269897460938, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.62903594970703, + "fcm_dpo/q_t": 0.4999913275241852, + "grad_norm": 0.0015491149388253689, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 1.2798230648040771, + "logits/rejected": 1.2812039852142334, + "logps/chosen": -261.1932373046875, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -358.9981384277344, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.62903594970703, + "margin_dpo/margin_std": 312.6791687011719, + "step": 425 + }, + { + "KL/chosen_KL_mean": -276.497314453125, + "KL/mean": -286.79400634765625, + "KL/rejected_KL_mean": -297.0906677246094, + "KL/std": 287.28729248046875, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 20.59334373474121, + "fcm_dpo/q_t": 0.4999980032444, + "grad_norm": 0.0016692699864506721, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 1.2584002017974854, + "logits/rejected": 1.2575640678405762, + "logps/chosen": -348.20550537109375, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -370.66790771484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 20.593345642089844, + "margin_dpo/margin_std": 402.493408203125, + "step": 426 + }, + { + "KL/chosen_KL_mean": -244.16412353515625, + "KL/mean": -299.6019287109375, + "KL/rejected_KL_mean": -355.03973388671875, + "KL/std": 328.6190185546875, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.87562561035156, + "fcm_dpo/q_t": 0.49998927116394043, + "grad_norm": 0.002250807359814644, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 1.2549666166305542, + "logits/rejected": 1.2232038974761963, + "logps/chosen": -302.806884765625, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -441.29412841796875, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.87562561035156, + "margin_dpo/margin_std": 329.8369140625, + "step": 427 + }, + { + "KL/chosen_KL_mean": -276.95233154296875, + "KL/mean": -329.34759521484375, + "KL/rejected_KL_mean": -381.74285888671875, + "KL/std": 323.8177185058594, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.79049682617188, + "fcm_dpo/q_t": 0.4999898374080658, + "grad_norm": 0.0020647216588258743, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 1.2424485683441162, + "logits/rejected": 1.2366441488265991, + "logps/chosen": -343.54840087890625, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -464.13702392578125, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.79049682617188, + "margin_dpo/margin_std": 409.09844970703125, + "step": 428 + }, + { + "KL/chosen_KL_mean": -245.20016479492188, + "KL/mean": -263.1362609863281, + "KL/rejected_KL_mean": -281.0723571777344, + "KL/std": 223.36959838867188, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 35.87220764160156, + "fcm_dpo/q_t": 0.499996542930603, + "grad_norm": 0.0011962910648435354, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 1.259268879890442, + "logits/rejected": 1.2742395401000977, + "logps/chosen": -302.2098693847656, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -340.9378662109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 35.8722038269043, + "margin_dpo/margin_std": 258.54791259765625, + "step": 429 + }, + { + "KL/chosen_KL_mean": -219.3946533203125, + "KL/mean": -255.57508850097656, + "KL/rejected_KL_mean": -291.75555419921875, + "KL/std": 203.72706604003906, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.36093139648438, + "fcm_dpo/q_t": 0.4999930262565613, + "grad_norm": 0.0014631770318374038, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 1.1872947216033936, + "logits/rejected": 1.187971591949463, + "logps/chosen": -278.95782470703125, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -362.2784423828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 72.36092376708984, + "margin_dpo/margin_std": 281.1922302246094, + "step": 430 + }, + { + "KL/chosen_KL_mean": -191.01605224609375, + "KL/mean": -245.7452850341797, + "KL/rejected_KL_mean": -300.47454833984375, + "KL/std": 234.70254516601562, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.45848083496094, + "fcm_dpo/q_t": 0.49998942017555237, + "grad_norm": 0.0020323162898421288, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 1.2493000030517578, + "logits/rejected": 1.2200860977172852, + "logps/chosen": -241.21636962890625, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -378.29132080078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.45848846435547, + "margin_dpo/margin_std": 307.2296142578125, + "step": 431 + }, + { + "KL/chosen_KL_mean": -256.33648681640625, + "KL/mean": -303.86322021484375, + "KL/rejected_KL_mean": -351.3900146484375, + "KL/std": 295.88214111328125, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.05351257324219, + "fcm_dpo/q_t": 0.4999908208847046, + "grad_norm": 0.0019163392717018723, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 1.286834716796875, + "logits/rejected": 1.2581818103790283, + "logps/chosen": -317.999267578125, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -435.3349609375, + "loss": 1.3863, + "margin_dpo/margin_mean": 95.05350494384766, + "margin_dpo/margin_std": 400.80767822265625, + "step": 432 + }, + { + "KL/chosen_KL_mean": -221.035400390625, + "KL/mean": -255.6205291748047, + "KL/rejected_KL_mean": -290.2056579589844, + "KL/std": 240.65435791015625, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 69.17024993896484, + "fcm_dpo/q_t": 0.49999332427978516, + "grad_norm": 0.0015163794159889221, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 1.273536205291748, + "logits/rejected": 1.2836341857910156, + "logps/chosen": -284.76458740234375, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -356.0447998046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 69.17024993896484, + "margin_dpo/margin_std": 295.4766845703125, + "step": 433 + }, + { + "KL/chosen_KL_mean": -177.51564025878906, + "KL/mean": -254.1595458984375, + "KL/rejected_KL_mean": -330.803466796875, + "KL/std": 256.6459655761719, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 153.2877960205078, + "fcm_dpo/q_t": 0.4999851584434509, + "grad_norm": 0.0024449967313557863, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 1.3231778144836426, + "logits/rejected": 1.300271987915039, + "logps/chosen": -225.48895263671875, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -403.31475830078125, + "loss": 1.3862, + "margin_dpo/margin_mean": 153.28781127929688, + "margin_dpo/margin_std": 307.38153076171875, + "step": 434 + }, + { + "KL/chosen_KL_mean": -212.97402954101562, + "KL/mean": -253.61410522460938, + "KL/rejected_KL_mean": -294.25421142578125, + "KL/std": 250.4377899169922, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 81.28021240234375, + "fcm_dpo/q_t": 0.49999213218688965, + "grad_norm": 0.0016701404238119721, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 1.3001594543457031, + "logits/rejected": 1.285398244857788, + "logps/chosen": -270.0342712402344, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -365.9456787109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 81.28020477294922, + "margin_dpo/margin_std": 294.3331298828125, + "step": 435 + }, + { + "KL/chosen_KL_mean": -216.45889282226562, + "KL/mean": -261.227294921875, + "KL/rejected_KL_mean": -305.9957275390625, + "KL/std": 265.82232666015625, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.53681182861328, + "fcm_dpo/q_t": 0.49999135732650757, + "grad_norm": 0.0022236332297325134, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 1.289149284362793, + "logits/rejected": 1.2721104621887207, + "logps/chosen": -272.616943359375, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -373.63360595703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.53681182861328, + "margin_dpo/margin_std": 242.168701171875, + "step": 436 + }, + { + "KL/chosen_KL_mean": -215.29190063476562, + "KL/mean": -292.97491455078125, + "KL/rejected_KL_mean": -370.65789794921875, + "KL/std": 285.3720397949219, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 155.36599731445312, + "fcm_dpo/q_t": 0.4999849498271942, + "grad_norm": 0.0029099148232489824, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 1.2747769355773926, + "logits/rejected": 1.217381238937378, + "logps/chosen": -272.2777099609375, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -456.2731628417969, + "loss": 1.3862, + "margin_dpo/margin_mean": 155.36599731445312, + "margin_dpo/margin_std": 385.7756042480469, + "step": 437 + }, + { + "KL/chosen_KL_mean": -179.04933166503906, + "KL/mean": -239.98414611816406, + "KL/rejected_KL_mean": -300.9189758300781, + "KL/std": 274.84893798828125, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.86964416503906, + "fcm_dpo/q_t": 0.4999881982803345, + "grad_norm": 0.002175640081986785, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 1.3229100704193115, + "logits/rejected": 1.2967147827148438, + "logps/chosen": -220.32711791992188, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -366.25738525390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.86964416503906, + "margin_dpo/margin_std": 330.1829528808594, + "step": 438 + }, + { + "KL/chosen_KL_mean": -285.81622314453125, + "KL/mean": -340.0804443359375, + "KL/rejected_KL_mean": -394.3446350097656, + "KL/std": 310.67144775390625, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.52841186523438, + "fcm_dpo/q_t": 0.49998950958251953, + "grad_norm": 0.0019142036326229572, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 1.188014268875122, + "logits/rejected": 1.1888413429260254, + "logps/chosen": -367.23388671875, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -489.0677185058594, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.52839660644531, + "margin_dpo/margin_std": 330.1500549316406, + "step": 439 + }, + { + "KL/chosen_KL_mean": -170.18858337402344, + "KL/mean": -235.81759643554688, + "KL/rejected_KL_mean": -301.44659423828125, + "KL/std": 246.02532958984375, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.25802612304688, + "fcm_dpo/q_t": 0.49998730421066284, + "grad_norm": 0.002329205861315131, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 1.2724201679229736, + "logits/rejected": 1.230487585067749, + "logps/chosen": -212.72677612304688, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -371.2347412109375, + "loss": 1.3862, + "margin_dpo/margin_mean": 131.25802612304688, + "margin_dpo/margin_std": 252.58546447753906, + "step": 440 + }, + { + "KL/chosen_KL_mean": -212.27427673339844, + "KL/mean": -289.75164794921875, + "KL/rejected_KL_mean": -367.22900390625, + "KL/std": 316.7421875, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.95472717285156, + "fcm_dpo/q_t": 0.499985009431839, + "grad_norm": 0.0029717073775827885, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 1.239859938621521, + "logits/rejected": 1.210754156112671, + "logps/chosen": -269.86749267578125, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -452.05780029296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 154.95472717285156, + "margin_dpo/margin_std": 341.307861328125, + "step": 441 + }, + { + "KL/chosen_KL_mean": -266.17144775390625, + "KL/mean": -336.0986328125, + "KL/rejected_KL_mean": -406.02587890625, + "KL/std": 319.74578857421875, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.8543701171875, + "fcm_dpo/q_t": 0.499986469745636, + "grad_norm": 0.0025126931723207235, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 1.2591302394866943, + "logits/rejected": 1.2514472007751465, + "logps/chosen": -333.6326599121094, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -495.09521484375, + "loss": 1.3862, + "margin_dpo/margin_mean": 139.85438537597656, + "margin_dpo/margin_std": 342.5337219238281, + "step": 442 + }, + { + "KL/chosen_KL_mean": -193.10202026367188, + "KL/mean": -278.1657409667969, + "KL/rejected_KL_mean": -363.22943115234375, + "KL/std": 328.15521240234375, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.12744140625, + "fcm_dpo/q_t": 0.499983549118042, + "grad_norm": 0.0025645981077104807, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 1.2513320446014404, + "logits/rejected": 1.2338635921478271, + "logps/chosen": -247.8981170654297, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -441.0372619628906, + "loss": 1.3862, + "margin_dpo/margin_mean": 170.12744140625, + "margin_dpo/margin_std": 411.92755126953125, + "step": 443 + }, + { + "KL/chosen_KL_mean": -238.94351196289062, + "KL/mean": -313.384521484375, + "KL/rejected_KL_mean": -387.8255615234375, + "KL/std": 340.74688720703125, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.88204956054688, + "fcm_dpo/q_t": 0.49998557567596436, + "grad_norm": 0.0032997538801282644, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 1.279803991317749, + "logits/rejected": 1.249694585800171, + "logps/chosen": -297.69256591796875, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -474.69952392578125, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.88204956054688, + "margin_dpo/margin_std": 396.679443359375, + "step": 444 + }, + { + "KL/chosen_KL_mean": -241.49159240722656, + "KL/mean": -290.87945556640625, + "KL/rejected_KL_mean": -340.2673645019531, + "KL/std": 260.7337341308594, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.77577209472656, + "fcm_dpo/q_t": 0.49999046325683594, + "grad_norm": 0.0014876207569614053, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 1.2597832679748535, + "logits/rejected": 1.2682100534439087, + "logps/chosen": -302.4090270996094, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -411.83111572265625, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.77577209472656, + "margin_dpo/margin_std": 304.1396484375, + "step": 445 + }, + { + "KL/chosen_KL_mean": -183.92437744140625, + "KL/mean": -257.93829345703125, + "KL/rejected_KL_mean": -331.9521484375, + "KL/std": 250.339111328125, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.02777099609375, + "fcm_dpo/q_t": 0.4999856650829315, + "grad_norm": 0.002402180340141058, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 1.2483906745910645, + "logits/rejected": 1.225560188293457, + "logps/chosen": -232.7236328125, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -403.8240966796875, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.02777099609375, + "margin_dpo/margin_std": 283.2513427734375, + "step": 446 + }, + { + "KL/chosen_KL_mean": -195.73277282714844, + "KL/mean": -288.6422424316406, + "KL/rejected_KL_mean": -381.55169677734375, + "KL/std": 270.0787353515625, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 185.81895446777344, + "fcm_dpo/q_t": 0.49998199939727783, + "grad_norm": 0.0031413903925567865, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 1.2971501350402832, + "logits/rejected": 1.2553091049194336, + "logps/chosen": -249.41549682617188, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -469.724853515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 185.81893920898438, + "margin_dpo/margin_std": 336.22564697265625, + "step": 447 + }, + { + "KL/chosen_KL_mean": -236.30259704589844, + "KL/mean": -296.205078125, + "KL/rejected_KL_mean": -356.1075439453125, + "KL/std": 286.213623046875, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.80496215820312, + "fcm_dpo/q_t": 0.4999884366989136, + "grad_norm": 0.002245934447273612, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 1.302297830581665, + "logits/rejected": 1.282482624053955, + "logps/chosen": -290.0538330078125, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -433.2837829589844, + "loss": 1.3862, + "margin_dpo/margin_mean": 119.80496215820312, + "margin_dpo/margin_std": 352.40093994140625, + "step": 448 + }, + { + "KL/chosen_KL_mean": -323.4554443359375, + "KL/mean": -345.5867919921875, + "KL/rejected_KL_mean": -367.7181701660156, + "KL/std": 359.06500244140625, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.262718200683594, + "fcm_dpo/q_t": 0.49999570846557617, + "grad_norm": 0.0019348947098478675, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 1.22648024559021, + "logits/rejected": 1.2325232028961182, + "logps/chosen": -399.2828369140625, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -449.925048828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 44.262718200683594, + "margin_dpo/margin_std": 400.443359375, + "step": 449 + }, + { + "KL/chosen_KL_mean": -181.9138946533203, + "KL/mean": -270.99798583984375, + "KL/rejected_KL_mean": -360.08209228515625, + "KL/std": 310.10675048828125, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.16824340820312, + "fcm_dpo/q_t": 0.4999827742576599, + "grad_norm": 0.0035590401384979486, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 1.3165409564971924, + "logits/rejected": 1.2763895988464355, + "logps/chosen": -229.02960205078125, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -438.8367919921875, + "loss": 1.3862, + "margin_dpo/margin_mean": 178.16824340820312, + "margin_dpo/margin_std": 364.7618408203125, + "step": 450 + }, + { + "KL/chosen_KL_mean": -227.50653076171875, + "KL/mean": -281.16180419921875, + "KL/rejected_KL_mean": -334.8171081542969, + "KL/std": 289.9800720214844, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.31055450439453, + "fcm_dpo/q_t": 0.4999896287918091, + "grad_norm": 0.002129608066752553, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 1.2388477325439453, + "logits/rejected": 1.225501298904419, + "logps/chosen": -290.85699462890625, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -411.1024169921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.31056213378906, + "margin_dpo/margin_std": 304.1910400390625, + "step": 451 + }, + { + "KL/chosen_KL_mean": -224.90155029296875, + "KL/mean": -285.91082763671875, + "KL/rejected_KL_mean": -346.92010498046875, + "KL/std": 273.02587890625, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.01853942871094, + "fcm_dpo/q_t": 0.4999881982803345, + "grad_norm": 0.002451541367918253, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 1.2452630996704102, + "logits/rejected": 1.216700553894043, + "logps/chosen": -280.4873962402344, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -424.60748291015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 122.01854705810547, + "margin_dpo/margin_std": 345.8335266113281, + "step": 452 + }, + { + "KL/chosen_KL_mean": -220.40438842773438, + "KL/mean": -267.64581298828125, + "KL/rejected_KL_mean": -314.8872375488281, + "KL/std": 251.55581665039062, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.48284149169922, + "fcm_dpo/q_t": 0.49999088048934937, + "grad_norm": 0.0015818747924640775, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 1.2517281770706177, + "logits/rejected": 1.2492544651031494, + "logps/chosen": -282.1825866699219, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -386.4012451171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 94.48284149169922, + "margin_dpo/margin_std": 269.7317810058594, + "step": 453 + }, + { + "KL/chosen_KL_mean": -197.87136840820312, + "KL/mean": -252.01133728027344, + "KL/rejected_KL_mean": -306.15130615234375, + "KL/std": 242.91844177246094, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.2799301147461, + "fcm_dpo/q_t": 0.49998950958251953, + "grad_norm": 0.001828188425861299, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 1.2671639919281006, + "logits/rejected": 1.2640118598937988, + "logps/chosen": -249.46652221679688, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -370.11865234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.27993774414062, + "margin_dpo/margin_std": 284.8143615722656, + "step": 454 + }, + { + "KL/chosen_KL_mean": -279.669677734375, + "KL/mean": -327.62640380859375, + "KL/rejected_KL_mean": -375.5830993652344, + "KL/std": 286.3306884765625, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.91340637207031, + "fcm_dpo/q_t": 0.49999070167541504, + "grad_norm": 0.0017985355807468295, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 1.2243902683258057, + "logits/rejected": 1.225219964981079, + "logps/chosen": -350.3214111328125, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -453.02587890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 95.91340637207031, + "margin_dpo/margin_std": 302.83038330078125, + "step": 455 + }, + { + "KL/chosen_KL_mean": -215.04669189453125, + "KL/mean": -274.91461181640625, + "KL/rejected_KL_mean": -334.78253173828125, + "KL/std": 239.27294921875, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.73584747314453, + "fcm_dpo/q_t": 0.4999884366989136, + "grad_norm": 0.0024633186403661966, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 1.2097394466400146, + "logits/rejected": 1.1886322498321533, + "logps/chosen": -271.4449768066406, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -417.3989562988281, + "loss": 1.3862, + "margin_dpo/margin_mean": 119.73583984375, + "margin_dpo/margin_std": 301.7488098144531, + "step": 456 + }, + { + "KL/chosen_KL_mean": -213.508544921875, + "KL/mean": -266.53350830078125, + "KL/rejected_KL_mean": -319.5584411621094, + "KL/std": 283.9473876953125, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.04988098144531, + "fcm_dpo/q_t": 0.49998974800109863, + "grad_norm": 0.0022488024551421404, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 1.3623318672180176, + "logits/rejected": 1.318712592124939, + "logps/chosen": -258.2291259765625, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -387.67431640625, + "loss": 1.3863, + "margin_dpo/margin_mean": 106.04988861083984, + "margin_dpo/margin_std": 333.2089538574219, + "step": 457 + }, + { + "KL/chosen_KL_mean": -188.3619384765625, + "KL/mean": -288.2777404785156, + "KL/rejected_KL_mean": -388.19354248046875, + "KL/std": 289.7039489746094, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 199.83157348632812, + "fcm_dpo/q_t": 0.4999806582927704, + "grad_norm": 0.00349022657610476, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 1.2601938247680664, + "logits/rejected": 1.2150036096572876, + "logps/chosen": -238.36764526367188, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -475.69366455078125, + "loss": 1.3862, + "margin_dpo/margin_mean": 199.83157348632812, + "margin_dpo/margin_std": 349.6432800292969, + "step": 458 + }, + { + "KL/chosen_KL_mean": -256.993896484375, + "KL/mean": -320.4532470703125, + "KL/rejected_KL_mean": -383.91259765625, + "KL/std": 375.80279541015625, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.91870880126953, + "fcm_dpo/q_t": 0.49998772144317627, + "grad_norm": 0.002192882588133216, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 1.2736434936523438, + "logits/rejected": 1.2665081024169922, + "logps/chosen": -322.371826171875, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -472.10504150390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.918701171875, + "margin_dpo/margin_std": 348.869140625, + "step": 459 + }, + { + "KL/chosen_KL_mean": -235.994384765625, + "KL/mean": -324.6895751953125, + "KL/rejected_KL_mean": -413.38482666015625, + "KL/std": 302.2080993652344, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 177.39044189453125, + "fcm_dpo/q_t": 0.4999828338623047, + "grad_norm": 0.00325951655395329, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 1.2858896255493164, + "logits/rejected": 1.2416361570358276, + "logps/chosen": -300.5560302734375, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -502.063720703125, + "loss": 1.3862, + "margin_dpo/margin_mean": 177.39044189453125, + "margin_dpo/margin_std": 368.12908935546875, + "step": 460 + }, + { + "KL/chosen_KL_mean": -190.54605102539062, + "KL/mean": -272.08990478515625, + "KL/rejected_KL_mean": -353.6337585449219, + "KL/std": 280.8585510253906, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 163.08770751953125, + "fcm_dpo/q_t": 0.4999842047691345, + "grad_norm": 0.0028130814898759127, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 1.2972333431243896, + "logits/rejected": 1.270198106765747, + "logps/chosen": -240.02395629882812, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -426.286376953125, + "loss": 1.3862, + "margin_dpo/margin_mean": 163.08770751953125, + "margin_dpo/margin_std": 284.0039978027344, + "step": 461 + }, + { + "KL/chosen_KL_mean": -188.55178833007812, + "KL/mean": -252.95437622070312, + "KL/rejected_KL_mean": -317.35693359375, + "KL/std": 175.87823486328125, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.80517578125, + "fcm_dpo/q_t": 0.49998754262924194, + "grad_norm": 0.002148764207959175, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 1.276878833770752, + "logits/rejected": 1.2633922100067139, + "logps/chosen": -249.04689025878906, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -392.1783142089844, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.80517578125, + "margin_dpo/margin_std": 224.19122314453125, + "step": 462 + }, + { + "KL/chosen_KL_mean": -273.58941650390625, + "KL/mean": -293.7682800292969, + "KL/rejected_KL_mean": -313.9471435546875, + "KL/std": 260.3634033203125, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 40.35773468017578, + "fcm_dpo/q_t": 0.4999960660934448, + "grad_norm": 0.0018327016150578856, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 1.2572622299194336, + "logits/rejected": 1.2583889961242676, + "logps/chosen": -341.2745361328125, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -385.26910400390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 40.35773468017578, + "margin_dpo/margin_std": 348.34228515625, + "step": 463 + }, + { + "KL/chosen_KL_mean": -230.39385986328125, + "KL/mean": -282.303466796875, + "KL/rejected_KL_mean": -334.21307373046875, + "KL/std": 285.50360107421875, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 103.8191909790039, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.0019776602275669575, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 1.2877094745635986, + "logits/rejected": 1.2803895473480225, + "logps/chosen": -289.55950927734375, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -403.7745361328125, + "loss": 1.3863, + "margin_dpo/margin_mean": 103.81919860839844, + "margin_dpo/margin_std": 348.489501953125, + "step": 464 + }, + { + "KL/chosen_KL_mean": -232.89459228515625, + "KL/mean": -298.0108337402344, + "KL/rejected_KL_mean": -363.1270751953125, + "KL/std": 265.3921813964844, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.23245239257812, + "fcm_dpo/q_t": 0.4999874234199524, + "grad_norm": 0.0023306291550397873, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 1.3072185516357422, + "logits/rejected": 1.2831568717956543, + "logps/chosen": -291.40826416015625, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -447.44451904296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.23245239257812, + "margin_dpo/margin_std": 334.58380126953125, + "step": 465 + }, + { + "KL/chosen_KL_mean": -299.5732421875, + "KL/mean": -299.2271728515625, + "KL/rejected_KL_mean": -298.881103515625, + "KL/std": 311.90045166015625, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.6921577453613281, + "fcm_dpo/q_t": 0.5000000596046448, + "grad_norm": 0.001482538296841085, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 1.261409878730774, + "logits/rejected": 1.2607641220092773, + "logps/chosen": -372.83905029296875, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -373.71728515625, + "loss": 1.3863, + "margin_dpo/margin_mean": -0.6921501159667969, + "margin_dpo/margin_std": 345.80474853515625, + "step": 466 + }, + { + "KL/chosen_KL_mean": -198.47096252441406, + "KL/mean": -302.1253356933594, + "KL/rejected_KL_mean": -405.77972412109375, + "KL/std": 344.73516845703125, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 207.30874633789062, + "fcm_dpo/q_t": 0.4999799430370331, + "grad_norm": 0.004299922846257687, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 1.2816591262817383, + "logits/rejected": 1.2331783771514893, + "logps/chosen": -246.05044555664062, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -484.4649353027344, + "loss": 1.3862, + "margin_dpo/margin_mean": 207.30874633789062, + "margin_dpo/margin_std": 427.2309265136719, + "step": 467 + }, + { + "KL/chosen_KL_mean": -241.93890380859375, + "KL/mean": -287.9136962890625, + "KL/rejected_KL_mean": -333.88848876953125, + "KL/std": 330.4139404296875, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.94956970214844, + "fcm_dpo/q_t": 0.4999910891056061, + "grad_norm": 0.0016830574022606015, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 1.2580692768096924, + "logits/rejected": 1.2581918239593506, + "logps/chosen": -305.86669921875, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -410.40472412109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.94956970214844, + "margin_dpo/margin_std": 405.082275390625, + "step": 468 + }, + { + "KL/chosen_KL_mean": -257.592529296875, + "KL/mean": -292.6340026855469, + "KL/rejected_KL_mean": -327.6754455566406, + "KL/std": 307.759521484375, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.08292388916016, + "fcm_dpo/q_t": 0.4999932050704956, + "grad_norm": 0.0019712212961167097, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 1.2918429374694824, + "logits/rejected": 1.2837411165237427, + "logps/chosen": -316.65069580078125, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -403.3521728515625, + "loss": 1.3863, + "margin_dpo/margin_mean": 70.08292388916016, + "margin_dpo/margin_std": 339.2579345703125, + "step": 469 + }, + { + "KL/chosen_KL_mean": -186.16371154785156, + "KL/mean": -246.60861206054688, + "KL/rejected_KL_mean": -307.05352783203125, + "KL/std": 228.65818786621094, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 120.88981628417969, + "fcm_dpo/q_t": 0.499988317489624, + "grad_norm": 0.002206693636253476, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 1.2765345573425293, + "logits/rejected": 1.256923794746399, + "logps/chosen": -234.03114318847656, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -373.0220947265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 120.88981628417969, + "margin_dpo/margin_std": 278.8352966308594, + "step": 470 + }, + { + "KL/chosen_KL_mean": -218.08656311035156, + "KL/mean": -278.8359680175781, + "KL/rejected_KL_mean": -339.5853576660156, + "KL/std": 286.42633056640625, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.498779296875, + "fcm_dpo/q_t": 0.49998825788497925, + "grad_norm": 0.0023722737096250057, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 1.238015055656433, + "logits/rejected": 1.219024658203125, + "logps/chosen": -275.8644104003906, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -413.3970947265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.498779296875, + "margin_dpo/margin_std": 337.14422607421875, + "step": 471 + }, + { + "KL/chosen_KL_mean": -214.7991485595703, + "KL/mean": -261.65557861328125, + "KL/rejected_KL_mean": -308.5119934082031, + "KL/std": 277.4007568359375, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 93.71286010742188, + "fcm_dpo/q_t": 0.49999094009399414, + "grad_norm": 0.0018467764602974057, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 1.2192950248718262, + "logits/rejected": 1.2010465860366821, + "logps/chosen": -270.70782470703125, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -383.2149353027344, + "loss": 1.3863, + "margin_dpo/margin_mean": 93.71286010742188, + "margin_dpo/margin_std": 368.9359130859375, + "step": 472 + }, + { + "KL/chosen_KL_mean": -204.37344360351562, + "KL/mean": -304.8914489746094, + "KL/rejected_KL_mean": -405.40948486328125, + "KL/std": 354.4217529296875, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 201.0360107421875, + "fcm_dpo/q_t": 0.49998053908348083, + "grad_norm": 0.003650173544883728, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 1.3033931255340576, + "logits/rejected": 1.2516118288040161, + "logps/chosen": -258.5343322753906, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -498.1773681640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 201.03599548339844, + "margin_dpo/margin_std": 407.78076171875, + "step": 473 + }, + { + "KL/chosen_KL_mean": -185.13568115234375, + "KL/mean": -263.2508544921875, + "KL/rejected_KL_mean": -341.36602783203125, + "KL/std": 283.1648254394531, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 156.2303466796875, + "fcm_dpo/q_t": 0.49998486042022705, + "grad_norm": 0.0029000849463045597, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 1.295190691947937, + "logits/rejected": 1.2742294073104858, + "logps/chosen": -231.82138061523438, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -412.8133544921875, + "loss": 1.3862, + "margin_dpo/margin_mean": 156.2303466796875, + "margin_dpo/margin_std": 273.30877685546875, + "step": 474 + }, + { + "KL/chosen_KL_mean": -244.22735595703125, + "KL/mean": -307.16156005859375, + "KL/rejected_KL_mean": -370.0957946777344, + "KL/std": 284.45855712890625, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 125.86846160888672, + "fcm_dpo/q_t": 0.4999878406524658, + "grad_norm": 0.002193914959207177, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 1.2467389106750488, + "logits/rejected": 1.2255311012268066, + "logps/chosen": -302.71466064453125, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -457.0976867675781, + "loss": 1.3862, + "margin_dpo/margin_mean": 125.86845397949219, + "margin_dpo/margin_std": 340.77777099609375, + "step": 475 + }, + { + "KL/chosen_KL_mean": -278.6739501953125, + "KL/mean": -314.0935363769531, + "KL/rejected_KL_mean": -349.51312255859375, + "KL/std": 314.4804382324219, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 70.83914184570312, + "fcm_dpo/q_t": 0.49999314546585083, + "grad_norm": 0.001390301389619708, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 1.2042824029922485, + "logits/rejected": 1.2116657495498657, + "logps/chosen": -354.05560302734375, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -426.5113525390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 70.83914184570312, + "margin_dpo/margin_std": 338.26055908203125, + "step": 476 + }, + { + "KL/chosen_KL_mean": -237.08114624023438, + "KL/mean": -295.21539306640625, + "KL/rejected_KL_mean": -353.34967041015625, + "KL/std": 313.53399658203125, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.26850891113281, + "fcm_dpo/q_t": 0.49998873472213745, + "grad_norm": 0.0024357279762625694, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 1.2554326057434082, + "logits/rejected": 1.2339025735855103, + "logps/chosen": -298.154541015625, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -434.69342041015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.26850891113281, + "margin_dpo/margin_std": 433.21453857421875, + "step": 477 + }, + { + "KL/chosen_KL_mean": -206.6198272705078, + "KL/mean": -230.17007446289062, + "KL/rejected_KL_mean": -253.7202911376953, + "KL/std": 195.4527130126953, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 47.100467681884766, + "fcm_dpo/q_t": 0.4999954402446747, + "grad_norm": 0.0013403815682977438, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 1.2230312824249268, + "logits/rejected": 1.2336747646331787, + "logps/chosen": -263.7871398925781, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -307.02947998046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 47.1004638671875, + "margin_dpo/margin_std": 252.40872192382812, + "step": 478 + }, + { + "KL/chosen_KL_mean": -214.67568969726562, + "KL/mean": -258.00732421875, + "KL/rejected_KL_mean": -301.3389892578125, + "KL/std": 248.02645874023438, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 86.66331481933594, + "fcm_dpo/q_t": 0.49999159574508667, + "grad_norm": 0.0024421438574790955, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 1.2256994247436523, + "logits/rejected": 1.2227530479431152, + "logps/chosen": -273.5889892578125, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -365.079345703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 86.66331481933594, + "margin_dpo/margin_std": 288.06201171875, + "step": 479 + }, + { + "KL/chosen_KL_mean": -232.02232360839844, + "KL/mean": -267.8770446777344, + "KL/rejected_KL_mean": -303.73175048828125, + "KL/std": 240.46432495117188, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.70942687988281, + "fcm_dpo/q_t": 0.49999308586120605, + "grad_norm": 0.0013092632871121168, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 1.234184741973877, + "logits/rejected": 1.2392432689666748, + "logps/chosen": -294.82293701171875, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -371.3203430175781, + "loss": 1.3863, + "margin_dpo/margin_mean": 71.70941925048828, + "margin_dpo/margin_std": 270.3521728515625, + "step": 480 + }, + { + "KL/chosen_KL_mean": -257.1699523925781, + "KL/mean": -295.2996826171875, + "KL/rejected_KL_mean": -333.42950439453125, + "KL/std": 286.9197082519531, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.25953674316406, + "fcm_dpo/q_t": 0.49999263882637024, + "grad_norm": 0.0014908494194969535, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 1.2494676113128662, + "logits/rejected": 1.257669448852539, + "logps/chosen": -322.4564208984375, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -404.2161865234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 76.2595443725586, + "margin_dpo/margin_std": 337.06689453125, + "step": 481 + }, + { + "KL/chosen_KL_mean": -263.18341064453125, + "KL/mean": -354.302978515625, + "KL/rejected_KL_mean": -445.4225158691406, + "KL/std": 386.18914794921875, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.2390594482422, + "fcm_dpo/q_t": 0.4999823570251465, + "grad_norm": 0.003877797396853566, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 1.230767011642456, + "logits/rejected": 1.1534785032272339, + "logps/chosen": -324.089599609375, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -548.8690795898438, + "loss": 1.3862, + "margin_dpo/margin_mean": 182.23907470703125, + "margin_dpo/margin_std": 488.2938232421875, + "step": 482 + }, + { + "KL/chosen_KL_mean": -213.94471740722656, + "KL/mean": -296.33514404296875, + "KL/rejected_KL_mean": -378.7255859375, + "KL/std": 284.4780578613281, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.78086853027344, + "fcm_dpo/q_t": 0.4999840557575226, + "grad_norm": 0.0031552868895232677, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 1.2734465599060059, + "logits/rejected": 1.2415012121200562, + "logps/chosen": -267.13671875, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -460.56488037109375, + "loss": 1.3862, + "margin_dpo/margin_mean": 164.78086853027344, + "margin_dpo/margin_std": 341.28466796875, + "step": 483 + }, + { + "KL/chosen_KL_mean": -244.69627380371094, + "KL/mean": -283.5141296386719, + "KL/rejected_KL_mean": -322.3320007324219, + "KL/std": 284.9932861328125, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.6357421875, + "fcm_dpo/q_t": 0.4999924898147583, + "grad_norm": 0.0018454925157129765, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 1.2972228527069092, + "logits/rejected": 1.2862370014190674, + "logps/chosen": -302.4657287597656, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -394.0149841308594, + "loss": 1.3863, + "margin_dpo/margin_mean": 77.6357421875, + "margin_dpo/margin_std": 290.02801513671875, + "step": 484 + }, + { + "KL/chosen_KL_mean": -232.70870971679688, + "KL/mean": -280.1761779785156, + "KL/rejected_KL_mean": -327.6436462402344, + "KL/std": 277.9726257324219, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.93495178222656, + "fcm_dpo/q_t": 0.4999908208847046, + "grad_norm": 0.0017475350759923458, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 1.2424696683883667, + "logits/rejected": 1.2406383752822876, + "logps/chosen": -289.34454345703125, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -398.4997863769531, + "loss": 1.3863, + "margin_dpo/margin_mean": 94.9349594116211, + "margin_dpo/margin_std": 307.6720275878906, + "step": 485 + }, + { + "KL/chosen_KL_mean": -219.482666015625, + "KL/mean": -304.8910827636719, + "KL/rejected_KL_mean": -390.29949951171875, + "KL/std": 327.8187255859375, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.81683349609375, + "fcm_dpo/q_t": 0.49998345971107483, + "grad_norm": 0.00316282594576478, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 1.260047435760498, + "logits/rejected": 1.2206038236618042, + "logps/chosen": -275.8296813964844, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -476.271728515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 170.8168487548828, + "margin_dpo/margin_std": 370.9302978515625, + "step": 486 + }, + { + "KL/chosen_KL_mean": -245.201416015625, + "KL/mean": -311.9969787597656, + "KL/rejected_KL_mean": -378.79254150390625, + "KL/std": 356.8734130859375, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 133.5911407470703, + "fcm_dpo/q_t": 0.49998706579208374, + "grad_norm": 0.0026520034298300743, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 1.3191529512405396, + "logits/rejected": 1.297263503074646, + "logps/chosen": -305.8186340332031, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -461.3022766113281, + "loss": 1.3862, + "margin_dpo/margin_mean": 133.59112548828125, + "margin_dpo/margin_std": 416.29364013671875, + "step": 487 + }, + { + "KL/chosen_KL_mean": -238.37918090820312, + "KL/mean": -309.9017333984375, + "KL/rejected_KL_mean": -381.42431640625, + "KL/std": 288.9468078613281, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 143.04513549804688, + "fcm_dpo/q_t": 0.4999861717224121, + "grad_norm": 0.0026024070102721453, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 1.2986162900924683, + "logits/rejected": 1.2784430980682373, + "logps/chosen": -301.48822021484375, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -463.91778564453125, + "loss": 1.3862, + "margin_dpo/margin_mean": 143.04513549804688, + "margin_dpo/margin_std": 348.89385986328125, + "step": 488 + }, + { + "KL/chosen_KL_mean": -275.52362060546875, + "KL/mean": -324.05682373046875, + "KL/rejected_KL_mean": -372.5899658203125, + "KL/std": 305.4133605957031, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.06634521484375, + "fcm_dpo/q_t": 0.4999906122684479, + "grad_norm": 0.0021042677108198404, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 1.2624127864837646, + "logits/rejected": 1.245917797088623, + "logps/chosen": -340.5125732421875, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -456.98602294921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 97.06634521484375, + "margin_dpo/margin_std": 349.16461181640625, + "step": 489 + }, + { + "KL/chosen_KL_mean": -234.01315307617188, + "KL/mean": -281.6185607910156, + "KL/rejected_KL_mean": -329.2239990234375, + "KL/std": 286.2935791015625, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.21084594726562, + "fcm_dpo/q_t": 0.4999907612800598, + "grad_norm": 0.001769617898389697, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 1.2830909490585327, + "logits/rejected": 1.288257360458374, + "logps/chosen": -295.921875, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -399.80963134765625, + "loss": 1.3863, + "margin_dpo/margin_mean": 95.21085357666016, + "margin_dpo/margin_std": 358.0466613769531, + "step": 490 + }, + { + "KL/chosen_KL_mean": -204.4666748046875, + "KL/mean": -257.9485778808594, + "KL/rejected_KL_mean": -311.43048095703125, + "KL/std": 246.65692138671875, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 106.96379852294922, + "fcm_dpo/q_t": 0.4999896287918091, + "grad_norm": 0.0019353864481672645, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 1.2297401428222656, + "logits/rejected": 1.202380895614624, + "logps/chosen": -259.9423828125, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -390.1336669921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 106.96379852294922, + "margin_dpo/margin_std": 309.4108581542969, + "step": 491 + }, + { + "KL/chosen_KL_mean": -274.214599609375, + "KL/mean": -337.2662353515625, + "KL/rejected_KL_mean": -400.31787109375, + "KL/std": 328.02960205078125, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.1032943725586, + "fcm_dpo/q_t": 0.49998778104782104, + "grad_norm": 0.002519700676202774, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 1.2626538276672363, + "logits/rejected": 1.2385413646697998, + "logps/chosen": -341.5009765625, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -483.10418701171875, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.1032943725586, + "margin_dpo/margin_std": 430.36181640625, + "step": 492 + }, + { + "KL/chosen_KL_mean": -193.55007934570312, + "KL/mean": -267.8668212890625, + "KL/rejected_KL_mean": -342.18353271484375, + "KL/std": 282.44659423828125, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.63345336914062, + "fcm_dpo/q_t": 0.49998563528060913, + "grad_norm": 0.0027879553381353617, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 1.2619478702545166, + "logits/rejected": 1.2343031167984009, + "logps/chosen": -249.47760009765625, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -421.3050537109375, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.63343811035156, + "margin_dpo/margin_std": 357.33514404296875, + "step": 493 + }, + { + "KL/chosen_KL_mean": -263.4035949707031, + "KL/mean": -321.43548583984375, + "KL/rejected_KL_mean": -379.4674072265625, + "KL/std": 349.48931884765625, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.06379699707031, + "fcm_dpo/q_t": 0.4999887943267822, + "grad_norm": 0.002763553289696574, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 1.210706114768982, + "logits/rejected": 1.1937711238861084, + "logps/chosen": -331.3576965332031, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -469.9760437011719, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.06378936767578, + "margin_dpo/margin_std": 462.98712158203125, + "step": 494 + }, + { + "KL/chosen_KL_mean": -195.76187133789062, + "KL/mean": -256.2266540527344, + "KL/rejected_KL_mean": -316.69140625, + "KL/std": 313.70343017578125, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 120.9295654296875, + "fcm_dpo/q_t": 0.49998828768730164, + "grad_norm": 0.002702318364754319, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 1.2848858833312988, + "logits/rejected": 1.2627146244049072, + "logps/chosen": -248.3873291015625, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -388.75921630859375, + "loss": 1.3862, + "margin_dpo/margin_mean": 120.92955780029297, + "margin_dpo/margin_std": 426.9603271484375, + "step": 495 + }, + { + "KL/chosen_KL_mean": -217.66781616210938, + "KL/mean": -307.02685546875, + "KL/rejected_KL_mean": -396.3858642578125, + "KL/std": 350.0010070800781, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 178.71804809570312, + "fcm_dpo/q_t": 0.49998271465301514, + "grad_norm": 0.003732392331585288, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 1.2494525909423828, + "logits/rejected": 1.1938560009002686, + "logps/chosen": -275.26513671875, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -490.7471618652344, + "loss": 1.3862, + "margin_dpo/margin_mean": 178.71804809570312, + "margin_dpo/margin_std": 492.61053466796875, + "step": 496 + }, + { + "KL/chosen_KL_mean": -261.3085021972656, + "KL/mean": -313.65008544921875, + "KL/rejected_KL_mean": -365.99163818359375, + "KL/std": 278.7204284667969, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.68314361572266, + "fcm_dpo/q_t": 0.4999898672103882, + "grad_norm": 0.002317574806511402, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 1.2523890733718872, + "logits/rejected": 1.24021315574646, + "logps/chosen": -334.09844970703125, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -455.4764709472656, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.68313598632812, + "margin_dpo/margin_std": 338.248291015625, + "step": 497 + }, + { + "KL/chosen_KL_mean": -224.85238647460938, + "KL/mean": -278.84197998046875, + "KL/rejected_KL_mean": -332.83160400390625, + "KL/std": 270.00634765625, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.97920989990234, + "fcm_dpo/q_t": 0.4999895691871643, + "grad_norm": 0.0020106714218854904, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 1.2464903593063354, + "logits/rejected": 1.2533658742904663, + "logps/chosen": -293.21807861328125, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -404.1200866699219, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.97921752929688, + "margin_dpo/margin_std": 352.33111572265625, + "step": 498 + }, + { + "KL/chosen_KL_mean": -242.342529296875, + "KL/mean": -309.7532653808594, + "KL/rejected_KL_mean": -377.1640319824219, + "KL/std": 302.4976806640625, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.82147216796875, + "fcm_dpo/q_t": 0.4999869465827942, + "grad_norm": 0.0024030758067965508, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 1.269078016281128, + "logits/rejected": 1.2355639934539795, + "logps/chosen": -304.2513427734375, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -469.10516357421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 134.82147216796875, + "margin_dpo/margin_std": 416.40582275390625, + "step": 499 + }, + { + "KL/chosen_KL_mean": -253.39688110351562, + "KL/mean": -294.63592529296875, + "KL/rejected_KL_mean": -335.87493896484375, + "KL/std": 297.90802001953125, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 82.47804260253906, + "fcm_dpo/q_t": 0.4999920129776001, + "grad_norm": 0.0017515573417767882, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 1.25115966796875, + "logits/rejected": 1.267160415649414, + "logps/chosen": -323.62274169921875, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -407.59698486328125, + "loss": 1.3863, + "margin_dpo/margin_mean": 82.47804260253906, + "margin_dpo/margin_std": 377.62408447265625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -261.0697326660156, + "KL/mean": -288.06939697265625, + "KL/rejected_KL_mean": -315.069091796875, + "KL/std": 263.5559387207031, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 53.99939727783203, + "fcm_dpo/q_t": 0.49999478459358215, + "grad_norm": 0.0011944005964323878, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 1.2886323928833008, + "logits/rejected": 1.289060354232788, + "logps/chosen": -325.66851806640625, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -385.66241455078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 53.99940490722656, + "margin_dpo/margin_std": 239.36679077148438, + "step": 501 + }, + { + "KL/chosen_KL_mean": -269.3963317871094, + "KL/mean": -318.6138916015625, + "KL/rejected_KL_mean": -367.83148193359375, + "KL/std": 324.662841796875, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.43511962890625, + "fcm_dpo/q_t": 0.49999046325683594, + "grad_norm": 0.0020151876378804445, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 1.2497098445892334, + "logits/rejected": 1.2299872636795044, + "logps/chosen": -334.86297607421875, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -458.0538024902344, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.43512725830078, + "margin_dpo/margin_std": 369.150634765625, + "step": 502 + }, + { + "KL/chosen_KL_mean": -190.64247131347656, + "KL/mean": -217.7227783203125, + "KL/rejected_KL_mean": -244.80307006835938, + "KL/std": 184.62762451171875, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 54.16061019897461, + "fcm_dpo/q_t": 0.49999475479125977, + "grad_norm": 0.0011236389400437474, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 1.2540428638458252, + "logits/rejected": 1.2642710208892822, + "logps/chosen": -242.47723388671875, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -302.4283142089844, + "loss": 1.3863, + "margin_dpo/margin_mean": 54.160614013671875, + "margin_dpo/margin_std": 232.08360290527344, + "step": 503 + }, + { + "KL/chosen_KL_mean": -230.22885131835938, + "KL/mean": -277.79840087890625, + "KL/rejected_KL_mean": -325.367919921875, + "KL/std": 244.9968719482422, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 95.13908386230469, + "fcm_dpo/q_t": 0.4999907910823822, + "grad_norm": 0.0015835659578442574, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 1.2332806587219238, + "logits/rejected": 1.2388975620269775, + "logps/chosen": -298.88006591796875, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -403.2818908691406, + "loss": 1.3863, + "margin_dpo/margin_mean": 95.13908386230469, + "margin_dpo/margin_std": 298.76800537109375, + "step": 504 + }, + { + "KL/chosen_KL_mean": -239.896484375, + "KL/mean": -291.92791748046875, + "KL/rejected_KL_mean": -343.9593200683594, + "KL/std": 342.92205810546875, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.06285095214844, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.002278943546116352, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 1.2496414184570312, + "logits/rejected": 1.2356963157653809, + "logps/chosen": -299.89532470703125, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -420.8398132324219, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.06285095214844, + "margin_dpo/margin_std": 399.6540222167969, + "step": 505 + }, + { + "KL/chosen_KL_mean": -271.675048828125, + "KL/mean": -315.27374267578125, + "KL/rejected_KL_mean": -358.87237548828125, + "KL/std": 294.46759033203125, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.19731903076172, + "fcm_dpo/q_t": 0.4999915361404419, + "grad_norm": 0.0017523845890536904, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 1.2550417184829712, + "logits/rejected": 1.2555642127990723, + "logps/chosen": -341.746337890625, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -440.9101257324219, + "loss": 1.3863, + "margin_dpo/margin_mean": 87.19731903076172, + "margin_dpo/margin_std": 383.368896484375, + "step": 506 + }, + { + "KL/chosen_KL_mean": -286.790283203125, + "KL/mean": -347.9385986328125, + "KL/rejected_KL_mean": -409.0868835449219, + "KL/std": 353.75408935546875, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 122.29658508300781, + "fcm_dpo/q_t": 0.4999881684780121, + "grad_norm": 0.0025757874827831984, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 1.2367452383041382, + "logits/rejected": 1.2185437679290771, + "logps/chosen": -358.79730224609375, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -503.0367431640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 122.29658508300781, + "margin_dpo/margin_std": 428.36688232421875, + "step": 507 + }, + { + "KL/chosen_KL_mean": -234.4129638671875, + "KL/mean": -341.6184387207031, + "KL/rejected_KL_mean": -448.82391357421875, + "KL/std": 360.1383056640625, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 214.41094970703125, + "fcm_dpo/q_t": 0.49997925758361816, + "grad_norm": 0.004092069808393717, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 1.2509568929672241, + "logits/rejected": 1.2035400867462158, + "logps/chosen": -294.63287353515625, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -544.7439575195312, + "loss": 1.3862, + "margin_dpo/margin_mean": 214.41094970703125, + "margin_dpo/margin_std": 459.3138122558594, + "step": 508 + }, + { + "KL/chosen_KL_mean": -234.1763458251953, + "KL/mean": -279.1103210449219, + "KL/rejected_KL_mean": -324.0443115234375, + "KL/std": 251.64889526367188, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.86798858642578, + "fcm_dpo/q_t": 0.4999912977218628, + "grad_norm": 0.001714211655780673, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 1.2467401027679443, + "logits/rejected": 1.2466963529586792, + "logps/chosen": -300.4465026855469, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -395.77496337890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.86798858642578, + "margin_dpo/margin_std": 300.79638671875, + "step": 509 + }, + { + "KL/chosen_KL_mean": -217.1124725341797, + "KL/mean": -314.4279479980469, + "KL/rejected_KL_mean": -411.743408203125, + "KL/std": 359.33087158203125, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 194.6309814453125, + "fcm_dpo/q_t": 0.499981164932251, + "grad_norm": 0.0035733687691390514, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 1.3153700828552246, + "logits/rejected": 1.2759437561035156, + "logps/chosen": -270.6573486328125, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -503.10992431640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 194.6309814453125, + "margin_dpo/margin_std": 484.9547119140625, + "step": 510 + }, + { + "KL/chosen_KL_mean": -246.47100830078125, + "KL/mean": -316.29608154296875, + "KL/rejected_KL_mean": -386.12115478515625, + "KL/std": 318.93328857421875, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.65016174316406, + "fcm_dpo/q_t": 0.499986469745636, + "grad_norm": 0.0027404821012169123, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 1.2811682224273682, + "logits/rejected": 1.257398009300232, + "logps/chosen": -302.31536865234375, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -472.6193542480469, + "loss": 1.3862, + "margin_dpo/margin_mean": 139.65016174316406, + "margin_dpo/margin_std": 339.78717041015625, + "step": 511 + }, + { + "KL/chosen_KL_mean": -258.85089111328125, + "KL/mean": -287.54278564453125, + "KL/rejected_KL_mean": -316.2346496582031, + "KL/std": 276.65423583984375, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.38376998901367, + "fcm_dpo/q_t": 0.4999944269657135, + "grad_norm": 0.0019444817444309592, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 1.254358172416687, + "logits/rejected": 1.2565571069717407, + "logps/chosen": -320.5039367675781, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -389.066162109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 57.383766174316406, + "margin_dpo/margin_std": 347.9620361328125, + "step": 512 + }, + { + "KL/chosen_KL_mean": -200.0421142578125, + "KL/mean": -261.5788269042969, + "KL/rejected_KL_mean": -323.11553955078125, + "KL/std": 258.5805969238281, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 123.07341003417969, + "fcm_dpo/q_t": 0.4999880790710449, + "grad_norm": 0.002271485747769475, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 1.3097870349884033, + "logits/rejected": 1.2971343994140625, + "logps/chosen": -250.89468383789062, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -392.33306884765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 123.07341003417969, + "margin_dpo/margin_std": 297.73004150390625, + "step": 513 + }, + { + "KL/chosen_KL_mean": -270.1363830566406, + "KL/mean": -320.6453552246094, + "KL/rejected_KL_mean": -371.15435791015625, + "KL/std": 278.06121826171875, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.01797485351562, + "fcm_dpo/q_t": 0.49999022483825684, + "grad_norm": 0.0019997938070446253, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 1.2445127964019775, + "logits/rejected": 1.241532802581787, + "logps/chosen": -339.52130126953125, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -454.47882080078125, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.01797485351562, + "margin_dpo/margin_std": 324.2949523925781, + "step": 514 + }, + { + "KL/chosen_KL_mean": -241.34022521972656, + "KL/mean": -301.35577392578125, + "KL/rejected_KL_mean": -361.371337890625, + "KL/std": 278.6277160644531, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 120.03112030029297, + "fcm_dpo/q_t": 0.4999883770942688, + "grad_norm": 0.0023619269486516714, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 1.2922251224517822, + "logits/rejected": 1.2539401054382324, + "logps/chosen": -295.02728271484375, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -444.96746826171875, + "loss": 1.3862, + "margin_dpo/margin_mean": 120.03111267089844, + "margin_dpo/margin_std": 330.4995422363281, + "step": 515 + }, + { + "KL/chosen_KL_mean": -212.0453643798828, + "KL/mean": -263.1299133300781, + "KL/rejected_KL_mean": -314.2144470214844, + "KL/std": 261.9797668457031, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.16908264160156, + "fcm_dpo/q_t": 0.4999901056289673, + "grad_norm": 0.0018633684376254678, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 1.2701607942581177, + "logits/rejected": 1.2603992223739624, + "logps/chosen": -268.94708251953125, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -382.0492248535156, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.16908264160156, + "margin_dpo/margin_std": 321.8263244628906, + "step": 516 + }, + { + "KL/chosen_KL_mean": -254.77786254882812, + "KL/mean": -277.159423828125, + "KL/rejected_KL_mean": -299.541015625, + "KL/std": 273.0831298828125, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 44.76313781738281, + "fcm_dpo/q_t": 0.4999956488609314, + "grad_norm": 0.001817610813304782, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 1.2920082807540894, + "logits/rejected": 1.2981328964233398, + "logps/chosen": -316.5530090332031, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -362.4237365722656, + "loss": 1.3863, + "margin_dpo/margin_mean": 44.76313400268555, + "margin_dpo/margin_std": 304.3832702636719, + "step": 517 + }, + { + "KL/chosen_KL_mean": -219.64892578125, + "KL/mean": -284.0128173828125, + "KL/rejected_KL_mean": -348.376708984375, + "KL/std": 307.4132385253906, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.727783203125, + "fcm_dpo/q_t": 0.49998754262924194, + "grad_norm": 0.002293572761118412, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 1.2533849477767944, + "logits/rejected": 1.2436450719833374, + "logps/chosen": -281.6741638183594, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -427.43756103515625, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.72779846191406, + "margin_dpo/margin_std": 379.77642822265625, + "step": 518 + }, + { + "KL/chosen_KL_mean": -248.82159423828125, + "KL/mean": -290.4149169921875, + "KL/rejected_KL_mean": -332.00823974609375, + "KL/std": 269.19866943359375, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 83.18663024902344, + "fcm_dpo/q_t": 0.4999919533729553, + "grad_norm": 0.002268343698233366, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 1.260763168334961, + "logits/rejected": 1.2478408813476562, + "logps/chosen": -310.427978515625, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -406.5155029296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 83.18663024902344, + "margin_dpo/margin_std": 341.28594970703125, + "step": 519 + }, + { + "KL/chosen_KL_mean": -225.19485473632812, + "KL/mean": -291.09344482421875, + "KL/rejected_KL_mean": -356.9920349121094, + "KL/std": 300.70770263671875, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.7971649169922, + "fcm_dpo/q_t": 0.49998724460601807, + "grad_norm": 0.002785892691463232, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 1.3261933326721191, + "logits/rejected": 1.3110194206237793, + "logps/chosen": -288.06829833984375, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -433.4976501464844, + "loss": 1.3862, + "margin_dpo/margin_mean": 131.7971649169922, + "margin_dpo/margin_std": 311.9014892578125, + "step": 520 + }, + { + "KL/chosen_KL_mean": -233.61395263671875, + "KL/mean": -314.4349060058594, + "KL/rejected_KL_mean": -395.255859375, + "KL/std": 295.2518310546875, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 161.64187622070312, + "fcm_dpo/q_t": 0.49998435378074646, + "grad_norm": 0.0028386113699525595, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 1.244354486465454, + "logits/rejected": 1.2204021215438843, + "logps/chosen": -297.82061767578125, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -487.53668212890625, + "loss": 1.3862, + "margin_dpo/margin_mean": 161.6418914794922, + "margin_dpo/margin_std": 372.21527099609375, + "step": 521 + }, + { + "KL/chosen_KL_mean": -254.05210876464844, + "KL/mean": -273.76922607421875, + "KL/rejected_KL_mean": -293.48638916015625, + "KL/std": 242.01705932617188, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 39.43425750732422, + "fcm_dpo/q_t": 0.4999961853027344, + "grad_norm": 0.001333137508481741, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 1.263893723487854, + "logits/rejected": 1.2616896629333496, + "logps/chosen": -312.42181396484375, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -362.27886962890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 39.434261322021484, + "margin_dpo/margin_std": 283.5724792480469, + "step": 522 + }, + { + "KL/chosen_KL_mean": -269.55242919921875, + "KL/mean": -339.4023132324219, + "KL/rejected_KL_mean": -409.252197265625, + "KL/std": 313.99249267578125, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 139.69979858398438, + "fcm_dpo/q_t": 0.499986469745636, + "grad_norm": 0.0031393070239573717, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 1.2486999034881592, + "logits/rejected": 1.2007946968078613, + "logps/chosen": -335.2656555175781, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -501.2411804199219, + "loss": 1.3862, + "margin_dpo/margin_mean": 139.69979858398438, + "margin_dpo/margin_std": 364.4853515625, + "step": 523 + }, + { + "KL/chosen_KL_mean": -258.45904541015625, + "KL/mean": -302.37017822265625, + "KL/rejected_KL_mean": -346.28131103515625, + "KL/std": 282.6915283203125, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 87.82229614257812, + "fcm_dpo/q_t": 0.4999915063381195, + "grad_norm": 0.0017077861120924354, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 1.2249318361282349, + "logits/rejected": 1.2157388925552368, + "logps/chosen": -334.8102722167969, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -436.2420349121094, + "loss": 1.3863, + "margin_dpo/margin_mean": 87.82229614257812, + "margin_dpo/margin_std": 297.6631774902344, + "step": 524 + }, + { + "KL/chosen_KL_mean": -292.2108154296875, + "KL/mean": -318.42254638671875, + "KL/rejected_KL_mean": -344.63427734375, + "KL/std": 277.3336486816406, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 52.42350387573242, + "fcm_dpo/q_t": 0.4999949336051941, + "grad_norm": 0.0015537795843556523, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 1.1935780048370361, + "logits/rejected": 1.1954379081726074, + "logps/chosen": -367.70660400390625, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -428.68280029296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 52.42350769042969, + "margin_dpo/margin_std": 347.2845458984375, + "step": 525 + }, + { + "KL/chosen_KL_mean": -247.38699340820312, + "KL/mean": -307.152587890625, + "KL/rejected_KL_mean": -366.91815185546875, + "KL/std": 318.8529968261719, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.53118896484375, + "fcm_dpo/q_t": 0.4999884366989136, + "grad_norm": 0.0027431962080299854, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 1.2542002201080322, + "logits/rejected": 1.226431131362915, + "logps/chosen": -308.67938232421875, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -449.39581298828125, + "loss": 1.3862, + "margin_dpo/margin_mean": 119.53117370605469, + "margin_dpo/margin_std": 372.6949768066406, + "step": 526 + }, + { + "KL/chosen_KL_mean": -241.8243408203125, + "KL/mean": -305.8216247558594, + "KL/rejected_KL_mean": -369.81884765625, + "KL/std": 304.98187255859375, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 127.9945068359375, + "fcm_dpo/q_t": 0.4999876022338867, + "grad_norm": 0.0031173720490187407, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 1.2762367725372314, + "logits/rejected": 1.2428832054138184, + "logps/chosen": -303.2969970703125, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -460.34716796875, + "loss": 1.3862, + "margin_dpo/margin_mean": 127.9945068359375, + "margin_dpo/margin_std": 330.25439453125, + "step": 527 + }, + { + "KL/chosen_KL_mean": -213.7490234375, + "KL/mean": -268.8468017578125, + "KL/rejected_KL_mean": -323.944580078125, + "KL/std": 265.69378662109375, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.19552612304688, + "fcm_dpo/q_t": 0.4999893605709076, + "grad_norm": 0.0022039057221263647, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 1.2133557796478271, + "logits/rejected": 1.192253589630127, + "logps/chosen": -272.5410461425781, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -395.76971435546875, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.1955337524414, + "margin_dpo/margin_std": 350.6766357421875, + "step": 528 + }, + { + "KL/chosen_KL_mean": -207.93161010742188, + "KL/mean": -281.9962158203125, + "KL/rejected_KL_mean": -356.06085205078125, + "KL/std": 229.11720275878906, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.1292266845703, + "fcm_dpo/q_t": 0.4999856650829315, + "grad_norm": 0.0023845250252634287, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 1.2498539686203003, + "logits/rejected": 1.2348082065582275, + "logps/chosen": -263.0025634765625, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -431.50091552734375, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.1292266845703, + "margin_dpo/margin_std": 242.88185119628906, + "step": 529 + }, + { + "KL/chosen_KL_mean": -232.45703125, + "KL/mean": -286.29730224609375, + "KL/rejected_KL_mean": -340.1375732421875, + "KL/std": 313.3363037109375, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 107.68051147460938, + "fcm_dpo/q_t": 0.4999895691871643, + "grad_norm": 0.002098920289427042, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 1.2603845596313477, + "logits/rejected": 1.2442349195480347, + "logps/chosen": -289.20086669921875, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -416.8067626953125, + "loss": 1.3863, + "margin_dpo/margin_mean": 107.68052673339844, + "margin_dpo/margin_std": 359.12908935546875, + "step": 530 + }, + { + "KL/chosen_KL_mean": -207.04556274414062, + "KL/mean": -264.3824462890625, + "KL/rejected_KL_mean": -321.7193603515625, + "KL/std": 270.8686218261719, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.67379760742188, + "fcm_dpo/q_t": 0.4999889135360718, + "grad_norm": 0.0024485730100423098, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 1.262247085571289, + "logits/rejected": 1.2219009399414062, + "logps/chosen": -258.1620178222656, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -401.2481689453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 114.67379760742188, + "margin_dpo/margin_std": 330.855712890625, + "step": 531 + }, + { + "KL/chosen_KL_mean": -252.53512573242188, + "KL/mean": -311.3507080078125, + "KL/rejected_KL_mean": -370.16632080078125, + "KL/std": 310.71435546875, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.63117980957031, + "fcm_dpo/q_t": 0.4999886155128479, + "grad_norm": 0.0020603465382009745, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 1.233213186264038, + "logits/rejected": 1.2175933122634888, + "logps/chosen": -310.8150634765625, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -448.2205810546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 117.63117980957031, + "margin_dpo/margin_std": 353.5859375, + "step": 532 + }, + { + "KL/chosen_KL_mean": -186.49844360351562, + "KL/mean": -257.16241455078125, + "KL/rejected_KL_mean": -327.8263854980469, + "KL/std": 312.2849426269531, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 141.3279571533203, + "fcm_dpo/q_t": 0.49998632073402405, + "grad_norm": 0.0025260313414037228, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 1.2709081172943115, + "logits/rejected": 1.2562451362609863, + "logps/chosen": -242.9164581298828, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -401.7196350097656, + "loss": 1.3862, + "margin_dpo/margin_mean": 141.3279571533203, + "margin_dpo/margin_std": 353.8362121582031, + "step": 533 + }, + { + "KL/chosen_KL_mean": -237.41729736328125, + "KL/mean": -282.9198913574219, + "KL/rejected_KL_mean": -328.4224853515625, + "KL/std": 276.8403625488281, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.00517272949219, + "fcm_dpo/q_t": 0.49999117851257324, + "grad_norm": 0.002167914994060993, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 1.2200748920440674, + "logits/rejected": 1.2163264751434326, + "logps/chosen": -298.1659851074219, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -402.2847900390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.00517272949219, + "margin_dpo/margin_std": 360.96624755859375, + "step": 534 + }, + { + "KL/chosen_KL_mean": -243.32843017578125, + "KL/mean": -293.2642822265625, + "KL/rejected_KL_mean": -343.20013427734375, + "KL/std": 254.28857421875, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.87169647216797, + "fcm_dpo/q_t": 0.4999903440475464, + "grad_norm": 0.0023620380088686943, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 1.2484328746795654, + "logits/rejected": 1.2265081405639648, + "logps/chosen": -304.9658508300781, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -424.13153076171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.8717041015625, + "margin_dpo/margin_std": 285.54364013671875, + "step": 535 + }, + { + "KL/chosen_KL_mean": -181.73826599121094, + "KL/mean": -265.3481140136719, + "KL/rejected_KL_mean": -348.95794677734375, + "KL/std": 246.2275390625, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 167.21969604492188, + "fcm_dpo/q_t": 0.4999838173389435, + "grad_norm": 0.003139457432553172, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 1.2669060230255127, + "logits/rejected": 1.2405221462249756, + "logps/chosen": -233.6272430419922, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -422.30657958984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 167.21969604492188, + "margin_dpo/margin_std": 304.58843994140625, + "step": 536 + }, + { + "KL/chosen_KL_mean": -222.4708251953125, + "KL/mean": -305.9654541015625, + "KL/rejected_KL_mean": -389.4600830078125, + "KL/std": 298.34405517578125, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 166.9892578125, + "fcm_dpo/q_t": 0.49998384714126587, + "grad_norm": 0.0030098857823759317, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 1.298835277557373, + "logits/rejected": 1.2577811479568481, + "logps/chosen": -276.7194519042969, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -484.40350341796875, + "loss": 1.3862, + "margin_dpo/margin_mean": 166.9892578125, + "margin_dpo/margin_std": 360.7626953125, + "step": 537 + }, + { + "KL/chosen_KL_mean": -288.58770751953125, + "KL/mean": -338.69049072265625, + "KL/rejected_KL_mean": -388.79327392578125, + "KL/std": 344.57171630859375, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 100.20556640625, + "fcm_dpo/q_t": 0.499990314245224, + "grad_norm": 0.002249750541523099, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 1.2919939756393433, + "logits/rejected": 1.285117268562317, + "logps/chosen": -358.6812438964844, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -468.2916259765625, + "loss": 1.3863, + "margin_dpo/margin_mean": 100.20558166503906, + "margin_dpo/margin_std": 351.8709411621094, + "step": 538 + }, + { + "KL/chosen_KL_mean": -225.29541015625, + "KL/mean": -294.580078125, + "KL/rejected_KL_mean": -363.8647766113281, + "KL/std": 290.5321044921875, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.56936645507812, + "fcm_dpo/q_t": 0.49998658895492554, + "grad_norm": 0.0030424855649471283, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 1.2382125854492188, + "logits/rejected": 1.2072124481201172, + "logps/chosen": -287.22711181640625, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -447.9542236328125, + "loss": 1.3862, + "margin_dpo/margin_mean": 138.56936645507812, + "margin_dpo/margin_std": 335.16851806640625, + "step": 539 + }, + { + "KL/chosen_KL_mean": -242.7515869140625, + "KL/mean": -336.0493469238281, + "KL/rejected_KL_mean": -429.34710693359375, + "KL/std": 340.7681884765625, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 186.59555053710938, + "fcm_dpo/q_t": 0.49998193979263306, + "grad_norm": 0.003513064468279481, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 1.3039121627807617, + "logits/rejected": 1.2665743827819824, + "logps/chosen": -305.455810546875, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -524.9830932617188, + "loss": 1.3862, + "margin_dpo/margin_mean": 186.59555053710938, + "margin_dpo/margin_std": 353.68701171875, + "step": 540 + }, + { + "KL/chosen_KL_mean": -221.798095703125, + "KL/mean": -250.46072387695312, + "KL/rejected_KL_mean": -279.1233825683594, + "KL/std": 239.862548828125, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.32527160644531, + "fcm_dpo/q_t": 0.4999944567680359, + "grad_norm": 0.0014312553685158491, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 1.290186882019043, + "logits/rejected": 1.3193426132202148, + "logps/chosen": -284.2789306640625, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -336.67877197265625, + "loss": 1.3863, + "margin_dpo/margin_mean": 57.32527160644531, + "margin_dpo/margin_std": 238.53594970703125, + "step": 541 + }, + { + "KL/chosen_KL_mean": -184.5562744140625, + "KL/mean": -236.01080322265625, + "KL/rejected_KL_mean": -287.46533203125, + "KL/std": 224.02711486816406, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.90904235839844, + "fcm_dpo/q_t": 0.4999900460243225, + "grad_norm": 0.0018258477794006467, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 1.2936313152313232, + "logits/rejected": 1.2742774486541748, + "logps/chosen": -234.01116943359375, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -352.798095703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.90904998779297, + "margin_dpo/margin_std": 270.83367919921875, + "step": 542 + }, + { + "KL/chosen_KL_mean": -195.629638671875, + "KL/mean": -286.21685791015625, + "KL/rejected_KL_mean": -376.8040771484375, + "KL/std": 335.13543701171875, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 181.17446899414062, + "fcm_dpo/q_t": 0.49998247623443604, + "grad_norm": 0.003246963955461979, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 1.257094383239746, + "logits/rejected": 1.228837490081787, + "logps/chosen": -246.73049926757812, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -452.86541748046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 181.1744842529297, + "margin_dpo/margin_std": 412.4785461425781, + "step": 543 + }, + { + "KL/chosen_KL_mean": -238.99652099609375, + "KL/mean": -313.9105529785156, + "KL/rejected_KL_mean": -388.8245544433594, + "KL/std": 292.81646728515625, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 149.82806396484375, + "fcm_dpo/q_t": 0.4999855160713196, + "grad_norm": 0.003041402902454138, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 1.2706331014633179, + "logits/rejected": 1.2441833019256592, + "logps/chosen": -299.27374267578125, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -477.2301025390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 149.8280487060547, + "margin_dpo/margin_std": 375.4241943359375, + "step": 544 + }, + { + "KL/chosen_KL_mean": -241.99879455566406, + "KL/mean": -287.8915100097656, + "KL/rejected_KL_mean": -333.7841796875, + "KL/std": 267.11920166015625, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.78538513183594, + "fcm_dpo/q_t": 0.49999111890792847, + "grad_norm": 0.0022016982547938824, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 1.2439241409301758, + "logits/rejected": 1.217350959777832, + "logps/chosen": -303.6140441894531, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -412.496826171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.78538513183594, + "margin_dpo/margin_std": 318.565673828125, + "step": 545 + }, + { + "KL/chosen_KL_mean": -221.55914306640625, + "KL/mean": -255.8831787109375, + "KL/rejected_KL_mean": -290.20721435546875, + "KL/std": 262.83538818359375, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.64806365966797, + "fcm_dpo/q_t": 0.49999332427978516, + "grad_norm": 0.0015192232094705105, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 1.302754282951355, + "logits/rejected": 1.3086268901824951, + "logps/chosen": -280.8724060058594, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -354.9435119628906, + "loss": 1.3863, + "margin_dpo/margin_mean": 68.6480712890625, + "margin_dpo/margin_std": 304.9132080078125, + "step": 546 + }, + { + "KL/chosen_KL_mean": -201.32754516601562, + "KL/mean": -267.2884826660156, + "KL/rejected_KL_mean": -333.24945068359375, + "KL/std": 234.06785583496094, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 131.92190551757812, + "fcm_dpo/q_t": 0.49998724460601807, + "grad_norm": 0.00217438698746264, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 1.2100231647491455, + "logits/rejected": 1.1951963901519775, + "logps/chosen": -256.3042907714844, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -408.608642578125, + "loss": 1.3862, + "margin_dpo/margin_mean": 131.92190551757812, + "margin_dpo/margin_std": 245.69558715820312, + "step": 547 + }, + { + "KL/chosen_KL_mean": -246.73431396484375, + "KL/mean": -302.9718017578125, + "KL/rejected_KL_mean": -359.20928955078125, + "KL/std": 298.39251708984375, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 112.47500610351562, + "fcm_dpo/q_t": 0.4999891221523285, + "grad_norm": 0.0023801657371222973, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 1.2188358306884766, + "logits/rejected": 1.203696846961975, + "logps/chosen": -309.9449768066406, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -440.4427795410156, + "loss": 1.3863, + "margin_dpo/margin_mean": 112.47501373291016, + "margin_dpo/margin_std": 335.791015625, + "step": 548 + }, + { + "KL/chosen_KL_mean": -246.50332641601562, + "KL/mean": -307.3105773925781, + "KL/rejected_KL_mean": -368.1178283691406, + "KL/std": 339.2923583984375, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.61450958251953, + "fcm_dpo/q_t": 0.49998822808265686, + "grad_norm": 0.00257733091711998, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 1.2848644256591797, + "logits/rejected": 1.2579424381256104, + "logps/chosen": -310.77685546875, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -460.4344482421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.61450958251953, + "margin_dpo/margin_std": 392.3224792480469, + "step": 549 + }, + { + "KL/chosen_KL_mean": -218.34458923339844, + "KL/mean": -256.52752685546875, + "KL/rejected_KL_mean": -294.71044921875, + "KL/std": 242.93511962890625, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.36582946777344, + "fcm_dpo/q_t": 0.49999260902404785, + "grad_norm": 0.0018148054368793964, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 1.2855148315429688, + "logits/rejected": 1.2823785543441772, + "logps/chosen": -274.57501220703125, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -357.3083190917969, + "loss": 1.3863, + "margin_dpo/margin_mean": 76.3658218383789, + "margin_dpo/margin_std": 288.6811218261719, + "step": 550 + }, + { + "KL/chosen_KL_mean": -248.3517303466797, + "KL/mean": -309.2677917480469, + "KL/rejected_KL_mean": -370.1838684082031, + "KL/std": 306.270751953125, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 121.83216094970703, + "fcm_dpo/q_t": 0.4999881982803345, + "grad_norm": 0.0023011192679405212, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 1.2524220943450928, + "logits/rejected": 1.2448160648345947, + "logps/chosen": -316.09893798828125, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -457.22674560546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 121.83216094970703, + "margin_dpo/margin_std": 341.08929443359375, + "step": 551 + }, + { + "KL/chosen_KL_mean": -232.7744140625, + "KL/mean": -290.295166015625, + "KL/rejected_KL_mean": -347.8158874511719, + "KL/std": 296.06097412109375, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.04149627685547, + "fcm_dpo/q_t": 0.4999888837337494, + "grad_norm": 0.0021020383574068546, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 1.2442529201507568, + "logits/rejected": 1.2225983142852783, + "logps/chosen": -295.70068359375, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -430.799560546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 115.04149627685547, + "margin_dpo/margin_std": 302.3883361816406, + "step": 552 + }, + { + "KL/chosen_KL_mean": -223.17971801757812, + "KL/mean": -299.51727294921875, + "KL/rejected_KL_mean": -375.8548583984375, + "KL/std": 325.5655517578125, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.67514038085938, + "fcm_dpo/q_t": 0.4999852180480957, + "grad_norm": 0.0032611123751848936, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 1.2355235815048218, + "logits/rejected": 1.1960480213165283, + "logps/chosen": -279.21820068359375, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -460.3393859863281, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.6751251220703, + "margin_dpo/margin_std": 377.767822265625, + "step": 553 + }, + { + "KL/chosen_KL_mean": -241.2162628173828, + "KL/mean": -287.3515625, + "KL/rejected_KL_mean": -333.48687744140625, + "KL/std": 261.48486328125, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 92.27063751220703, + "fcm_dpo/q_t": 0.4999910593032837, + "grad_norm": 0.0016775699332356453, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 1.247375726699829, + "logits/rejected": 1.2545955181121826, + "logps/chosen": -305.7468566894531, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -404.7025146484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 92.27063751220703, + "margin_dpo/margin_std": 262.7148742675781, + "step": 554 + }, + { + "KL/chosen_KL_mean": -242.61669921875, + "KL/mean": -278.1309814453125, + "KL/rejected_KL_mean": -313.645263671875, + "KL/std": 235.6993408203125, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 71.02857971191406, + "fcm_dpo/q_t": 0.49999311566352844, + "grad_norm": 0.0012565868673846126, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 1.2524785995483398, + "logits/rejected": 1.2564916610717773, + "logps/chosen": -309.26861572265625, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -382.3119812011719, + "loss": 1.3863, + "margin_dpo/margin_mean": 71.02857971191406, + "margin_dpo/margin_std": 269.95416259765625, + "step": 555 + }, + { + "KL/chosen_KL_mean": -205.25283813476562, + "KL/mean": -254.348876953125, + "KL/rejected_KL_mean": -303.4449462890625, + "KL/std": 214.45046997070312, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.19207763671875, + "fcm_dpo/q_t": 0.49999046325683594, + "grad_norm": 0.001899282680824399, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 1.2756855487823486, + "logits/rejected": 1.2640533447265625, + "logps/chosen": -258.085205078125, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -367.93536376953125, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.19209289550781, + "margin_dpo/margin_std": 222.29449462890625, + "step": 556 + }, + { + "KL/chosen_KL_mean": -227.02926635742188, + "KL/mean": -313.59149169921875, + "KL/rejected_KL_mean": -400.1536865234375, + "KL/std": 321.18719482421875, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 173.12440490722656, + "fcm_dpo/q_t": 0.4999832510948181, + "grad_norm": 0.003363779978826642, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 1.310492992401123, + "logits/rejected": 1.288170576095581, + "logps/chosen": -282.06524658203125, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -475.96014404296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 173.12442016601562, + "margin_dpo/margin_std": 381.8446350097656, + "step": 557 + }, + { + "KL/chosen_KL_mean": -248.1762237548828, + "KL/mean": -312.669189453125, + "KL/rejected_KL_mean": -377.16217041015625, + "KL/std": 295.711669921875, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 128.98593139648438, + "fcm_dpo/q_t": 0.49998751282691956, + "grad_norm": 0.0022693185601383448, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 1.2714734077453613, + "logits/rejected": 1.2482508420944214, + "logps/chosen": -311.402587890625, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -468.6309814453125, + "loss": 1.3862, + "margin_dpo/margin_mean": 128.98593139648438, + "margin_dpo/margin_std": 365.5770263671875, + "step": 558 + }, + { + "KL/chosen_KL_mean": -236.52352905273438, + "KL/mean": -312.6187744140625, + "KL/rejected_KL_mean": -388.71405029296875, + "KL/std": 306.2005615234375, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.19052124023438, + "fcm_dpo/q_t": 0.4999852776527405, + "grad_norm": 0.0024620580952614546, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 1.2394496202468872, + "logits/rejected": 1.2185451984405518, + "logps/chosen": -298.045166015625, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -471.5526428222656, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.1905059814453, + "margin_dpo/margin_std": 371.47833251953125, + "step": 559 + }, + { + "KL/chosen_KL_mean": -240.15634155273438, + "KL/mean": -291.6183776855469, + "KL/rejected_KL_mean": -343.08038330078125, + "KL/std": 260.51849365234375, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.924072265625, + "fcm_dpo/q_t": 0.4999900460243225, + "grad_norm": 0.002105724299326539, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 1.3063299655914307, + "logits/rejected": 1.2882559299468994, + "logps/chosen": -300.79754638671875, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -421.83514404296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.924072265625, + "margin_dpo/margin_std": 279.7295227050781, + "step": 560 + }, + { + "KL/chosen_KL_mean": -231.18161010742188, + "KL/mean": -277.0013427734375, + "KL/rejected_KL_mean": -322.821044921875, + "KL/std": 236.68751525878906, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.63945770263672, + "fcm_dpo/q_t": 0.49999114871025085, + "grad_norm": 0.0019574423786252737, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 1.2543569803237915, + "logits/rejected": 1.2415952682495117, + "logps/chosen": -293.68023681640625, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -401.54168701171875, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.63945770263672, + "margin_dpo/margin_std": 286.7827453613281, + "step": 561 + }, + { + "KL/chosen_KL_mean": -279.95501708984375, + "KL/mean": -352.92193603515625, + "KL/rejected_KL_mean": -425.88885498046875, + "KL/std": 363.0885009765625, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 145.93382263183594, + "fcm_dpo/q_t": 0.49998587369918823, + "grad_norm": 0.002555001061409712, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 1.256950855255127, + "logits/rejected": 1.2576011419296265, + "logps/chosen": -354.73675537109375, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -518.5238037109375, + "loss": 1.3862, + "margin_dpo/margin_mean": 145.93382263183594, + "margin_dpo/margin_std": 366.14715576171875, + "step": 562 + }, + { + "KL/chosen_KL_mean": -186.45376586914062, + "KL/mean": -236.38587951660156, + "KL/rejected_KL_mean": -286.3179931640625, + "KL/std": 214.30816650390625, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.86424255371094, + "fcm_dpo/q_t": 0.4999903440475464, + "grad_norm": 0.0017422273522242904, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 1.329614281654358, + "logits/rejected": 1.3034515380859375, + "logps/chosen": -236.6522674560547, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -353.0848693847656, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.86424255371094, + "margin_dpo/margin_std": 297.21343994140625, + "step": 563 + }, + { + "KL/chosen_KL_mean": -198.27935791015625, + "KL/mean": -253.13412475585938, + "KL/rejected_KL_mean": -307.9888610839844, + "KL/std": 239.6055145263672, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.70948028564453, + "fcm_dpo/q_t": 0.49998939037323, + "grad_norm": 0.002160602016374469, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 1.2637401819229126, + "logits/rejected": 1.250982642173767, + "logps/chosen": -254.0202178955078, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -382.81207275390625, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.70947265625, + "margin_dpo/margin_std": 332.83740234375, + "step": 564 + }, + { + "KL/chosen_KL_mean": -264.5958557128906, + "KL/mean": -321.555908203125, + "KL/rejected_KL_mean": -378.5159912109375, + "KL/std": 297.69415283203125, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 113.92015075683594, + "fcm_dpo/q_t": 0.49998897314071655, + "grad_norm": 0.002675557043403387, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 1.3032476902008057, + "logits/rejected": 1.2836694717407227, + "logps/chosen": -322.9332275390625, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -456.833740234375, + "loss": 1.3863, + "margin_dpo/margin_mean": 113.92015075683594, + "margin_dpo/margin_std": 335.97607421875, + "step": 565 + }, + { + "KL/chosen_KL_mean": -251.61575317382812, + "KL/mean": -298.7529296875, + "KL/rejected_KL_mean": -345.89007568359375, + "KL/std": 310.39801025390625, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.27432250976562, + "fcm_dpo/q_t": 0.49999088048934937, + "grad_norm": 0.0015177810564637184, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 1.2642093896865845, + "logits/rejected": 1.2712171077728271, + "logps/chosen": -322.8394775390625, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -417.006103515625, + "loss": 1.3863, + "margin_dpo/margin_mean": 94.27432250976562, + "margin_dpo/margin_std": 391.6617431640625, + "step": 566 + }, + { + "KL/chosen_KL_mean": -201.491943359375, + "KL/mean": -286.70196533203125, + "KL/rejected_KL_mean": -371.9119873046875, + "KL/std": 300.2903747558594, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 170.42001342773438, + "fcm_dpo/q_t": 0.4999834895133972, + "grad_norm": 0.002934924094006419, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 1.2649750709533691, + "logits/rejected": 1.2357361316680908, + "logps/chosen": -254.1612091064453, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -446.25982666015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 170.42002868652344, + "margin_dpo/margin_std": 392.0222473144531, + "step": 567 + }, + { + "KL/chosen_KL_mean": -201.22650146484375, + "KL/mean": -296.65826416015625, + "KL/rejected_KL_mean": -392.0899963378906, + "KL/std": 302.1213073730469, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 190.86349487304688, + "fcm_dpo/q_t": 0.49998152256011963, + "grad_norm": 0.003850526176393032, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 1.2919840812683105, + "logits/rejected": 1.2600350379943848, + "logps/chosen": -253.4044952392578, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -477.9177551269531, + "loss": 1.3862, + "margin_dpo/margin_mean": 190.86349487304688, + "margin_dpo/margin_std": 386.70355224609375, + "step": 568 + }, + { + "KL/chosen_KL_mean": -245.7794189453125, + "KL/mean": -283.582275390625, + "KL/rejected_KL_mean": -321.3851318359375, + "KL/std": 323.57220458984375, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 75.60567474365234, + "fcm_dpo/q_t": 0.4999926686286926, + "grad_norm": 0.0020971940830349922, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 1.2494065761566162, + "logits/rejected": 1.2419638633728027, + "logps/chosen": -308.4286804199219, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -396.81494140625, + "loss": 1.3863, + "margin_dpo/margin_mean": 75.60567474365234, + "margin_dpo/margin_std": 389.1278381347656, + "step": 569 + }, + { + "KL/chosen_KL_mean": -213.00286865234375, + "KL/mean": -280.4457702636719, + "KL/rejected_KL_mean": -347.888671875, + "KL/std": 267.0771789550781, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 134.8858184814453, + "fcm_dpo/q_t": 0.4999869465827942, + "grad_norm": 0.0022408810909837484, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 1.2825806140899658, + "logits/rejected": 1.2437119483947754, + "logps/chosen": -263.044677734375, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -426.16015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 134.88583374023438, + "margin_dpo/margin_std": 310.875, + "step": 570 + }, + { + "KL/chosen_KL_mean": -230.393310546875, + "KL/mean": -274.6932678222656, + "KL/rejected_KL_mean": -318.99322509765625, + "KL/std": 289.2368469238281, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.59991455078125, + "fcm_dpo/q_t": 0.49999141693115234, + "grad_norm": 0.0022626707796007395, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 1.280132532119751, + "logits/rejected": 1.2787832021713257, + "logps/chosen": -284.05010986328125, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -385.126220703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 88.59992980957031, + "margin_dpo/margin_std": 364.80145263671875, + "step": 571 + }, + { + "KL/chosen_KL_mean": -287.53131103515625, + "KL/mean": -291.213623046875, + "KL/rejected_KL_mean": -294.89599609375, + "KL/std": 277.07366943359375, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.364675521850586, + "fcm_dpo/q_t": 0.4999992847442627, + "grad_norm": 0.001692647230811417, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 1.258278250694275, + "logits/rejected": 1.2805957794189453, + "logps/chosen": -362.3492431640625, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -360.7828063964844, + "loss": 1.3863, + "margin_dpo/margin_mean": 7.364673614501953, + "margin_dpo/margin_std": 344.2757568359375, + "step": 572 + }, + { + "KL/chosen_KL_mean": -281.252685546875, + "KL/mean": -336.67999267578125, + "KL/rejected_KL_mean": -392.10736083984375, + "KL/std": 340.51861572265625, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 110.85468292236328, + "fcm_dpo/q_t": 0.4999893009662628, + "grad_norm": 0.002122138626873493, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 1.2949762344360352, + "logits/rejected": 1.2791988849639893, + "logps/chosen": -349.97833251953125, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -480.26934814453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 110.85468292236328, + "margin_dpo/margin_std": 395.3820495605469, + "step": 573 + }, + { + "KL/chosen_KL_mean": -224.64120483398438, + "KL/mean": -303.6715087890625, + "KL/rejected_KL_mean": -382.7017822265625, + "KL/std": 301.89447021484375, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 158.06060791015625, + "fcm_dpo/q_t": 0.4999846816062927, + "grad_norm": 0.0023631516378372908, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 1.285936713218689, + "logits/rejected": 1.2656428813934326, + "logps/chosen": -280.95458984375, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -466.6173095703125, + "loss": 1.3862, + "margin_dpo/margin_mean": 158.0605926513672, + "margin_dpo/margin_std": 359.18701171875, + "step": 574 + }, + { + "KL/chosen_KL_mean": -240.16212463378906, + "KL/mean": -306.2142333984375, + "KL/rejected_KL_mean": -372.2663269042969, + "KL/std": 327.98846435546875, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 132.1042022705078, + "fcm_dpo/q_t": 0.4999872148036957, + "grad_norm": 0.00271924352273345, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 1.2739206552505493, + "logits/rejected": 1.2386579513549805, + "logps/chosen": -304.74627685546875, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -465.7366638183594, + "loss": 1.3862, + "margin_dpo/margin_mean": 132.1042022705078, + "margin_dpo/margin_std": 406.8048095703125, + "step": 575 + }, + { + "KL/chosen_KL_mean": -185.20242309570312, + "KL/mean": -287.28302001953125, + "KL/rejected_KL_mean": -389.36358642578125, + "KL/std": 299.2473449707031, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 204.16119384765625, + "fcm_dpo/q_t": 0.49998024106025696, + "grad_norm": 0.0033409777097404003, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 1.2817810773849487, + "logits/rejected": 1.2451581954956055, + "logps/chosen": -238.48294067382812, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -473.56365966796875, + "loss": 1.3862, + "margin_dpo/margin_mean": 204.16119384765625, + "margin_dpo/margin_std": 395.5772705078125, + "step": 576 + }, + { + "KL/chosen_KL_mean": -230.99880981445312, + "KL/mean": -275.36846923828125, + "KL/rejected_KL_mean": -319.7381591796875, + "KL/std": 301.64703369140625, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 88.7393569946289, + "fcm_dpo/q_t": 0.49999138712882996, + "grad_norm": 0.002223484218120575, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 1.2827671766281128, + "logits/rejected": 1.2928658723831177, + "logps/chosen": -293.323486328125, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -387.0386962890625, + "loss": 1.3863, + "margin_dpo/margin_mean": 88.73936462402344, + "margin_dpo/margin_std": 334.8846435546875, + "step": 577 + }, + { + "KL/chosen_KL_mean": -238.35610961914062, + "KL/mean": -270.2100830078125, + "KL/rejected_KL_mean": -302.0640869140625, + "KL/std": 326.62493896484375, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 63.7080078125, + "fcm_dpo/q_t": 0.49999380111694336, + "grad_norm": 0.0015426744939759374, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 1.3032362461090088, + "logits/rejected": 1.3008297681808472, + "logps/chosen": -295.0116882324219, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -370.282470703125, + "loss": 1.3863, + "margin_dpo/margin_mean": 63.7080078125, + "margin_dpo/margin_std": 349.22344970703125, + "step": 578 + }, + { + "KL/chosen_KL_mean": -219.2360076904297, + "KL/mean": -270.0174255371094, + "KL/rejected_KL_mean": -320.7988586425781, + "KL/std": 243.76840209960938, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.56283569335938, + "fcm_dpo/q_t": 0.49999016523361206, + "grad_norm": 0.001675861538387835, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 1.258755087852478, + "logits/rejected": 1.2550430297851562, + "logps/chosen": -276.0456848144531, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -388.8949890136719, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.56283569335938, + "margin_dpo/margin_std": 307.77728271484375, + "step": 579 + }, + { + "KL/chosen_KL_mean": -229.52764892578125, + "KL/mean": -310.90203857421875, + "KL/rejected_KL_mean": -392.2764587402344, + "KL/std": 354.71600341796875, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 162.74879455566406, + "fcm_dpo/q_t": 0.4999842643737793, + "grad_norm": 0.0027315111365169287, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 1.252476453781128, + "logits/rejected": 1.2458863258361816, + "logps/chosen": -287.227783203125, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -470.18310546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 162.74879455566406, + "margin_dpo/margin_std": 423.9322204589844, + "step": 580 + }, + { + "KL/chosen_KL_mean": -243.62631225585938, + "KL/mean": -305.4908447265625, + "KL/rejected_KL_mean": -367.3553466796875, + "KL/std": 283.1879577636719, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 123.72905731201172, + "fcm_dpo/q_t": 0.49998801946640015, + "grad_norm": 0.002351470058783889, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 1.2822654247283936, + "logits/rejected": 1.2544565200805664, + "logps/chosen": -302.95867919921875, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -451.00018310546875, + "loss": 1.3862, + "margin_dpo/margin_mean": 123.72904968261719, + "margin_dpo/margin_std": 322.47772216796875, + "step": 581 + }, + { + "KL/chosen_KL_mean": -234.3776092529297, + "KL/mean": -258.0681457519531, + "KL/rejected_KL_mean": -281.7586669921875, + "KL/std": 228.97186279296875, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 47.381065368652344, + "fcm_dpo/q_t": 0.4999954104423523, + "grad_norm": 0.0013384189223870635, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 1.285982370376587, + "logits/rejected": 1.3120609521865845, + "logps/chosen": -298.54046630859375, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -340.3916015625, + "loss": 1.3863, + "margin_dpo/margin_mean": 47.38106918334961, + "margin_dpo/margin_std": 315.0589904785156, + "step": 582 + }, + { + "KL/chosen_KL_mean": -187.56515502929688, + "KL/mean": -304.24945068359375, + "KL/rejected_KL_mean": -420.93377685546875, + "KL/std": 375.9952087402344, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 233.36862182617188, + "fcm_dpo/q_t": 0.4999774098396301, + "grad_norm": 0.0039016844239085913, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 1.322705626487732, + "logits/rejected": 1.2900676727294922, + "logps/chosen": -239.4375457763672, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -504.7970886230469, + "loss": 1.3862, + "margin_dpo/margin_mean": 233.36859130859375, + "margin_dpo/margin_std": 436.15399169921875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -171.99374389648438, + "KL/mean": -252.2381591796875, + "KL/rejected_KL_mean": -332.48260498046875, + "KL/std": 261.9239807128906, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 160.48883056640625, + "fcm_dpo/q_t": 0.499984472990036, + "grad_norm": 0.003471849486231804, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 1.3081614971160889, + "logits/rejected": 1.2692360877990723, + "logps/chosen": -218.5651397705078, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -413.16229248046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 160.48883056640625, + "margin_dpo/margin_std": 291.8946228027344, + "step": 584 + }, + { + "KL/chosen_KL_mean": -242.6631622314453, + "KL/mean": -276.41943359375, + "KL/rejected_KL_mean": -310.1756896972656, + "KL/std": 275.22735595703125, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.51254272460938, + "fcm_dpo/q_t": 0.4999934732913971, + "grad_norm": 0.001750921132043004, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 1.2745296955108643, + "logits/rejected": 1.2461663484573364, + "logps/chosen": -300.7877197265625, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -389.18109130859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 67.51254272460938, + "margin_dpo/margin_std": 317.5940246582031, + "step": 585 + }, + { + "KL/chosen_KL_mean": -195.4407958984375, + "KL/mean": -253.28018188476562, + "KL/rejected_KL_mean": -311.11956787109375, + "KL/std": 251.697021484375, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 115.67878723144531, + "fcm_dpo/q_t": 0.4999887943267822, + "grad_norm": 0.0022813198156654835, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 1.2979401350021362, + "logits/rejected": 1.2939859628677368, + "logps/chosen": -249.54241943359375, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -374.8406982421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 115.67877960205078, + "margin_dpo/margin_std": 308.5834655761719, + "step": 586 + }, + { + "KL/chosen_KL_mean": -257.64990234375, + "KL/mean": -289.8480224609375, + "KL/rejected_KL_mean": -322.046142578125, + "KL/std": 334.7420959472656, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 64.39622497558594, + "fcm_dpo/q_t": 0.4999937415122986, + "grad_norm": 0.0018675602041184902, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 1.2932814359664917, + "logits/rejected": 1.2983059883117676, + "logps/chosen": -321.0671081542969, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -385.5162048339844, + "loss": 1.3863, + "margin_dpo/margin_mean": 64.39622497558594, + "margin_dpo/margin_std": 453.559814453125, + "step": 587 + }, + { + "KL/chosen_KL_mean": -256.47723388671875, + "KL/mean": -315.72064208984375, + "KL/rejected_KL_mean": -374.96405029296875, + "KL/std": 312.8167724609375, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 118.48678588867188, + "fcm_dpo/q_t": 0.4999885559082031, + "grad_norm": 0.0022324968595057726, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 1.307366132736206, + "logits/rejected": 1.2943463325500488, + "logps/chosen": -318.67828369140625, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -457.0665283203125, + "loss": 1.3862, + "margin_dpo/margin_mean": 118.48678588867188, + "margin_dpo/margin_std": 407.2249755859375, + "step": 588 + }, + { + "KL/chosen_KL_mean": -240.05502319335938, + "KL/mean": -276.25872802734375, + "KL/rejected_KL_mean": -312.46246337890625, + "KL/std": 287.0700378417969, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 72.40746307373047, + "fcm_dpo/q_t": 0.4999929666519165, + "grad_norm": 0.001997460378333926, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 1.2659380435943604, + "logits/rejected": 1.2496306896209717, + "logps/chosen": -296.76861572265625, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -389.19915771484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 72.40746307373047, + "margin_dpo/margin_std": 305.57440185546875, + "step": 589 + }, + { + "KL/chosen_KL_mean": -246.395263671875, + "KL/mean": -291.65802001953125, + "KL/rejected_KL_mean": -336.9208068847656, + "KL/std": 262.71783447265625, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.52552032470703, + "fcm_dpo/q_t": 0.499991238117218, + "grad_norm": 0.002160015283152461, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 1.2421910762786865, + "logits/rejected": 1.2292957305908203, + "logps/chosen": -312.9090881347656, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -422.6290283203125, + "loss": 1.3863, + "margin_dpo/margin_mean": 90.52552795410156, + "margin_dpo/margin_std": 272.5157470703125, + "step": 590 + }, + { + "KL/chosen_KL_mean": -197.66827392578125, + "KL/mean": -279.355224609375, + "KL/rejected_KL_mean": -361.042236328125, + "KL/std": 297.4459228515625, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 163.37396240234375, + "fcm_dpo/q_t": 0.4999842047691345, + "grad_norm": 0.0024286925327032804, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 1.2815182209014893, + "logits/rejected": 1.266688585281372, + "logps/chosen": -258.3654479980469, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -447.1650390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 163.37396240234375, + "margin_dpo/margin_std": 392.95819091796875, + "step": 591 + }, + { + "KL/chosen_KL_mean": -201.080322265625, + "KL/mean": -273.27728271484375, + "KL/rejected_KL_mean": -345.4742736816406, + "KL/std": 250.97198486328125, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 144.39395141601562, + "fcm_dpo/q_t": 0.4999859929084778, + "grad_norm": 0.0026749190874397755, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 1.2822060585021973, + "logits/rejected": 1.2332913875579834, + "logps/chosen": -252.31764221191406, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -427.0766906738281, + "loss": 1.3862, + "margin_dpo/margin_mean": 144.39395141601562, + "margin_dpo/margin_std": 278.5047607421875, + "step": 592 + }, + { + "KL/chosen_KL_mean": -165.1071319580078, + "KL/mean": -229.7209014892578, + "KL/rejected_KL_mean": -294.33465576171875, + "KL/std": 204.59747314453125, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 129.22750854492188, + "fcm_dpo/q_t": 0.49998748302459717, + "grad_norm": 0.002245598705485463, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 1.2915213108062744, + "logits/rejected": 1.2537872791290283, + "logps/chosen": -207.1871337890625, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -362.80963134765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 129.22750854492188, + "margin_dpo/margin_std": 228.52536010742188, + "step": 593 + }, + { + "KL/chosen_KL_mean": -243.26272583007812, + "KL/mean": -281.5732421875, + "KL/rejected_KL_mean": -319.88372802734375, + "KL/std": 282.73321533203125, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.62098693847656, + "fcm_dpo/q_t": 0.49999260902404785, + "grad_norm": 0.0013759155990555882, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 1.2534310817718506, + "logits/rejected": 1.2528616189956665, + "logps/chosen": -306.92138671875, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -390.23968505859375, + "loss": 1.3863, + "margin_dpo/margin_mean": 76.62098693847656, + "margin_dpo/margin_std": 327.9439697265625, + "step": 594 + }, + { + "KL/chosen_KL_mean": -198.01931762695312, + "KL/mean": -272.7323913574219, + "KL/rejected_KL_mean": -347.44549560546875, + "KL/std": 273.14923095703125, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 149.4261474609375, + "fcm_dpo/q_t": 0.4999855160713196, + "grad_norm": 0.0027599758468568325, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 1.3034560680389404, + "logits/rejected": 1.2654231786727905, + "logps/chosen": -254.2380828857422, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -431.4031982421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 149.4261474609375, + "margin_dpo/margin_std": 291.680908203125, + "step": 595 + }, + { + "KL/chosen_KL_mean": -268.5811767578125, + "KL/mean": -279.02777099609375, + "KL/rejected_KL_mean": -289.474365234375, + "KL/std": 269.3888244628906, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 20.893169403076172, + "fcm_dpo/q_t": 0.49999797344207764, + "grad_norm": 0.001436872873455286, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 1.215804100036621, + "logits/rejected": 1.2338910102844238, + "logps/chosen": -337.06207275390625, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -351.20733642578125, + "loss": 1.3863, + "margin_dpo/margin_mean": 20.893165588378906, + "margin_dpo/margin_std": 239.36436462402344, + "step": 596 + }, + { + "KL/chosen_KL_mean": -190.21514892578125, + "KL/mean": -211.99703979492188, + "KL/rejected_KL_mean": -233.7789306640625, + "KL/std": 190.4124755859375, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 43.563785552978516, + "fcm_dpo/q_t": 0.49999576807022095, + "grad_norm": 0.0016540532233193517, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 1.276237964630127, + "logits/rejected": 1.267054557800293, + "logps/chosen": -239.07266235351562, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -288.8470153808594, + "loss": 1.3863, + "margin_dpo/margin_mean": 43.56378173828125, + "margin_dpo/margin_std": 183.7689208984375, + "step": 597 + }, + { + "KL/chosen_KL_mean": -242.9158935546875, + "KL/mean": -291.18408203125, + "KL/rejected_KL_mean": -339.4522705078125, + "KL/std": 272.9280090332031, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.53633880615234, + "fcm_dpo/q_t": 0.49999064207077026, + "grad_norm": 0.0021814981009811163, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 1.2822654247283936, + "logits/rejected": 1.2468507289886475, + "logps/chosen": -301.80303955078125, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -420.88372802734375, + "loss": 1.3863, + "margin_dpo/margin_mean": 96.53633117675781, + "margin_dpo/margin_std": 309.15411376953125, + "step": 598 + }, + { + "KL/chosen_KL_mean": -241.9964599609375, + "KL/mean": -287.59735107421875, + "KL/rejected_KL_mean": -333.1982421875, + "KL/std": 273.47198486328125, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.2018051147461, + "fcm_dpo/q_t": 0.49999117851257324, + "grad_norm": 0.001978973625227809, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 1.3052904605865479, + "logits/rejected": 1.2893433570861816, + "logps/chosen": -299.6036376953125, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -405.0029296875, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.20179748535156, + "margin_dpo/margin_std": 326.43243408203125, + "step": 599 + }, + { + "KL/chosen_KL_mean": -229.87130737304688, + "KL/mean": -281.87884521484375, + "KL/rejected_KL_mean": -333.8863525390625, + "KL/std": 281.9407958984375, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.01502227783203, + "fcm_dpo/q_t": 0.49998992681503296, + "grad_norm": 0.00212049949914217, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 1.2342650890350342, + "logits/rejected": 1.2097012996673584, + "logps/chosen": -288.3136291503906, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -417.53271484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.01502227783203, + "margin_dpo/margin_std": 371.31842041015625, + "step": 600 + }, + { + "KL/chosen_KL_mean": -204.21005249023438, + "KL/mean": -272.1788635253906, + "KL/rejected_KL_mean": -340.147705078125, + "KL/std": 289.89752197265625, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 135.9376220703125, + "fcm_dpo/q_t": 0.499986857175827, + "grad_norm": 0.002290458418428898, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 1.2456085681915283, + "logits/rejected": 1.209405541419983, + "logps/chosen": -259.80438232421875, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -423.833984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 135.9376220703125, + "margin_dpo/margin_std": 325.400634765625, + "step": 601 + }, + { + "KL/chosen_KL_mean": -210.0914764404297, + "KL/mean": -256.0052185058594, + "KL/rejected_KL_mean": -301.9189453125, + "KL/std": 259.4412841796875, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 91.82748413085938, + "fcm_dpo/q_t": 0.49999111890792847, + "grad_norm": 0.002178723691031337, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 1.2279410362243652, + "logits/rejected": 1.2131808996200562, + "logps/chosen": -266.440673828125, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -373.9149169921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 91.82748413085938, + "margin_dpo/margin_std": 305.39068603515625, + "step": 602 + }, + { + "KL/chosen_KL_mean": -200.24911499023438, + "KL/mean": -269.5469665527344, + "KL/rejected_KL_mean": -338.8448486328125, + "KL/std": 285.01885986328125, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.59573364257812, + "fcm_dpo/q_t": 0.49998658895492554, + "grad_norm": 0.0024140651803463697, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 1.2549041509628296, + "logits/rejected": 1.244372844696045, + "logps/chosen": -253.41749572753906, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -412.705322265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 138.59573364257812, + "margin_dpo/margin_std": 345.45501708984375, + "step": 603 + }, + { + "KL/chosen_KL_mean": -227.04763793945312, + "KL/mean": -257.6112365722656, + "KL/rejected_KL_mean": -288.1748046875, + "KL/std": 230.52130126953125, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 61.12716293334961, + "fcm_dpo/q_t": 0.49999406933784485, + "grad_norm": 0.0016172666801139712, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 1.210902214050293, + "logits/rejected": 1.2236418724060059, + "logps/chosen": -299.69708251953125, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -358.0540771484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 61.127159118652344, + "margin_dpo/margin_std": 327.2840576171875, + "step": 604 + }, + { + "KL/chosen_KL_mean": -239.35777282714844, + "KL/mean": -294.0518798828125, + "KL/rejected_KL_mean": -348.7460632324219, + "KL/std": 237.30853271484375, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 109.38829040527344, + "fcm_dpo/q_t": 0.49998942017555237, + "grad_norm": 0.002217090455815196, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 1.2849633693695068, + "logits/rejected": 1.2634557485580444, + "logps/chosen": -300.9706115722656, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -428.09002685546875, + "loss": 1.3863, + "margin_dpo/margin_mean": 109.38829040527344, + "margin_dpo/margin_std": 277.03143310546875, + "step": 605 + }, + { + "KL/chosen_KL_mean": -208.34193420410156, + "KL/mean": -277.22564697265625, + "KL/rejected_KL_mean": -346.10943603515625, + "KL/std": 279.93414306640625, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 137.76748657226562, + "fcm_dpo/q_t": 0.4999866485595703, + "grad_norm": 0.00246420968323946, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 1.2400498390197754, + "logits/rejected": 1.221116542816162, + "logps/chosen": -262.80615234375, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -425.73651123046875, + "loss": 1.3862, + "margin_dpo/margin_mean": 137.76748657226562, + "margin_dpo/margin_std": 318.7607727050781, + "step": 606 + }, + { + "KL/chosen_KL_mean": -250.8245391845703, + "KL/mean": -299.26092529296875, + "KL/rejected_KL_mean": -347.69732666015625, + "KL/std": 316.63677978515625, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 96.87279510498047, + "fcm_dpo/q_t": 0.49999064207077026, + "grad_norm": 0.0016430618707090616, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 1.2330322265625, + "logits/rejected": 1.2289059162139893, + "logps/chosen": -313.6854248046875, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -420.2475280761719, + "loss": 1.3863, + "margin_dpo/margin_mean": 96.87281036376953, + "margin_dpo/margin_std": 384.65234375, + "step": 607 + }, + { + "KL/chosen_KL_mean": -240.77908325195312, + "KL/mean": -311.6595458984375, + "KL/rejected_KL_mean": -382.5400390625, + "KL/std": 341.69573974609375, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 141.76089477539062, + "fcm_dpo/q_t": 0.49998629093170166, + "grad_norm": 0.002434444148093462, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 1.211251974105835, + "logits/rejected": 1.174188494682312, + "logps/chosen": -303.9598083496094, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -481.69891357421875, + "loss": 1.3862, + "margin_dpo/margin_mean": 141.7609100341797, + "margin_dpo/margin_std": 400.3000793457031, + "step": 608 + }, + { + "KL/chosen_KL_mean": -180.31060791015625, + "KL/mean": -238.63388061523438, + "KL/rejected_KL_mean": -296.9571533203125, + "KL/std": 239.36196899414062, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.64653015136719, + "fcm_dpo/q_t": 0.49998870491981506, + "grad_norm": 0.001932095387019217, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 1.271007776260376, + "logits/rejected": 1.262525200843811, + "logps/chosen": -228.933837890625, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -365.2398681640625, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.64653015136719, + "margin_dpo/margin_std": 259.3739318847656, + "step": 609 + }, + { + "KL/chosen_KL_mean": -303.2109680175781, + "KL/mean": -337.4134521484375, + "KL/rejected_KL_mean": -371.615966796875, + "KL/std": 341.649658203125, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 68.40501403808594, + "fcm_dpo/q_t": 0.49999338388442993, + "grad_norm": 0.0016822778852656484, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 1.248294711112976, + "logits/rejected": 1.2520670890808105, + "logps/chosen": -375.8760986328125, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -458.7690734863281, + "loss": 1.3863, + "margin_dpo/margin_mean": 68.40501403808594, + "margin_dpo/margin_std": 410.22003173828125, + "step": 610 + }, + { + "KL/chosen_KL_mean": -202.11181640625, + "KL/mean": -256.2706604003906, + "KL/rejected_KL_mean": -310.42950439453125, + "KL/std": 272.47833251953125, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.31767272949219, + "fcm_dpo/q_t": 0.49998950958251953, + "grad_norm": 0.0027471468783915043, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 1.3028149604797363, + "logits/rejected": 1.2825827598571777, + "logps/chosen": -250.42039489746094, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -381.0436096191406, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.31765747070312, + "margin_dpo/margin_std": 253.734375, + "step": 611 + }, + { + "KL/chosen_KL_mean": -241.52801513671875, + "KL/mean": -312.7408447265625, + "KL/rejected_KL_mean": -383.95361328125, + "KL/std": 254.8455810546875, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 142.42562866210938, + "fcm_dpo/q_t": 0.4999862313270569, + "grad_norm": 0.0026474539190530777, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 1.2259531021118164, + "logits/rejected": 1.1953879594802856, + "logps/chosen": -302.75958251953125, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -478.33343505859375, + "loss": 1.3862, + "margin_dpo/margin_mean": 142.42559814453125, + "margin_dpo/margin_std": 282.14910888671875, + "step": 612 + }, + { + "KL/chosen_KL_mean": -215.14187622070312, + "KL/mean": -248.74453735351562, + "KL/rejected_KL_mean": -282.34716796875, + "KL/std": 317.31195068359375, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 67.20530700683594, + "fcm_dpo/q_t": 0.4999935030937195, + "grad_norm": 0.0015928486827760935, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 1.2564080953598022, + "logits/rejected": 1.26479971408844, + "logps/chosen": -269.125, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -340.66925048828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 67.20530700683594, + "margin_dpo/margin_std": 282.4429626464844, + "step": 613 + }, + { + "KL/chosen_KL_mean": -235.35379028320312, + "KL/mean": -268.393798828125, + "KL/rejected_KL_mean": -301.4338073730469, + "KL/std": 251.66497802734375, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 66.08002471923828, + "fcm_dpo/q_t": 0.49999362230300903, + "grad_norm": 0.0015550341922789812, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 1.275475025177002, + "logits/rejected": 1.2758376598358154, + "logps/chosen": -295.5968322753906, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -373.6964111328125, + "loss": 1.3863, + "margin_dpo/margin_mean": 66.08003234863281, + "margin_dpo/margin_std": 330.7596435546875, + "step": 614 + }, + { + "KL/chosen_KL_mean": -288.22857666015625, + "KL/mean": -350.49859619140625, + "KL/rejected_KL_mean": -412.76861572265625, + "KL/std": 364.74468994140625, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 124.53999328613281, + "fcm_dpo/q_t": 0.49998795986175537, + "grad_norm": 0.0024001309648156166, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 1.2872897386550903, + "logits/rejected": 1.2527488470077515, + "logps/chosen": -360.3232727050781, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -516.7984008789062, + "loss": 1.3862, + "margin_dpo/margin_mean": 124.53999328613281, + "margin_dpo/margin_std": 407.09320068359375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -214.70602416992188, + "KL/mean": -280.04010009765625, + "KL/rejected_KL_mean": -345.3741455078125, + "KL/std": 338.1444091796875, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.6681365966797, + "fcm_dpo/q_t": 0.4999873638153076, + "grad_norm": 0.002586161717772484, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 1.3120701313018799, + "logits/rejected": 1.2948602437973022, + "logps/chosen": -273.23675537109375, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -420.85443115234375, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.6681365966797, + "margin_dpo/margin_std": 311.08026123046875, + "step": 616 + }, + { + "KL/chosen_KL_mean": -231.34402465820312, + "KL/mean": -275.96807861328125, + "KL/rejected_KL_mean": -320.59210205078125, + "KL/std": 250.75045776367188, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.24805450439453, + "fcm_dpo/q_t": 0.49999135732650757, + "grad_norm": 0.0019385057967156172, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 1.3058488368988037, + "logits/rejected": 1.2810614109039307, + "logps/chosen": -288.95269775390625, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -401.81317138671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 89.24806213378906, + "margin_dpo/margin_std": 294.38568115234375, + "step": 617 + }, + { + "KL/chosen_KL_mean": -219.1179656982422, + "KL/mean": -298.6544189453125, + "KL/rejected_KL_mean": -378.19085693359375, + "KL/std": 302.0130310058594, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 159.0728759765625, + "fcm_dpo/q_t": 0.49998462200164795, + "grad_norm": 0.003072551218792796, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 1.2910728454589844, + "logits/rejected": 1.2586588859558105, + "logps/chosen": -275.81390380859375, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -464.1144714355469, + "loss": 1.3862, + "margin_dpo/margin_mean": 159.0728759765625, + "margin_dpo/margin_std": 343.8255920410156, + "step": 618 + }, + { + "KL/chosen_KL_mean": -202.87046813964844, + "KL/mean": -281.81170654296875, + "KL/rejected_KL_mean": -360.75299072265625, + "KL/std": 335.5523681640625, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 157.8825225830078, + "fcm_dpo/q_t": 0.4999847114086151, + "grad_norm": 0.002635043114423752, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 1.328965425491333, + "logits/rejected": 1.297224760055542, + "logps/chosen": -256.92889404296875, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -444.30792236328125, + "loss": 1.3862, + "margin_dpo/margin_mean": 157.8825225830078, + "margin_dpo/margin_std": 431.3692626953125, + "step": 619 + }, + { + "KL/chosen_KL_mean": -240.50149536132812, + "KL/mean": -257.7876281738281, + "KL/rejected_KL_mean": -275.0737609863281, + "KL/std": 267.0464782714844, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 34.57228088378906, + "fcm_dpo/q_t": 0.4999966323375702, + "grad_norm": 0.0017283577471971512, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 1.2990144491195679, + "logits/rejected": 1.3006207942962646, + "logps/chosen": -303.8712158203125, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -340.7564392089844, + "loss": 1.3863, + "margin_dpo/margin_mean": 34.57228088378906, + "margin_dpo/margin_std": 325.6532287597656, + "step": 620 + }, + { + "KL/chosen_KL_mean": -205.8194580078125, + "KL/mean": -296.57342529296875, + "KL/rejected_KL_mean": -387.327392578125, + "KL/std": 351.635498046875, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 181.50796508789062, + "fcm_dpo/q_t": 0.49998241662979126, + "grad_norm": 0.0033599112648516893, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 1.2818564176559448, + "logits/rejected": 1.236130952835083, + "logps/chosen": -258.14068603515625, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -475.41741943359375, + "loss": 1.3862, + "margin_dpo/margin_mean": 181.50796508789062, + "margin_dpo/margin_std": 466.3568115234375, + "step": 621 + }, + { + "KL/chosen_KL_mean": -252.88168334960938, + "KL/mean": -307.3717956542969, + "KL/rejected_KL_mean": -361.8619384765625, + "KL/std": 309.87310791015625, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 108.98025512695312, + "fcm_dpo/q_t": 0.49998944997787476, + "grad_norm": 0.0022065993398427963, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 1.2621958255767822, + "logits/rejected": 1.2421605587005615, + "logps/chosen": -312.74713134765625, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -443.7286071777344, + "loss": 1.3863, + "margin_dpo/margin_mean": 108.98025512695312, + "margin_dpo/margin_std": 352.85650634765625, + "step": 622 + }, + { + "KL/chosen_KL_mean": -274.98095703125, + "KL/mean": -340.2034606933594, + "KL/rejected_KL_mean": -405.4259338378906, + "KL/std": 373.55999755859375, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 130.4449462890625, + "fcm_dpo/q_t": 0.4999873638153076, + "grad_norm": 0.002324384171515703, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 1.2612360715866089, + "logits/rejected": 1.245605707168579, + "logps/chosen": -342.34942626953125, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -487.4532470703125, + "loss": 1.3862, + "margin_dpo/margin_mean": 130.4449462890625, + "margin_dpo/margin_std": 484.2095642089844, + "step": 623 + }, + { + "KL/chosen_KL_mean": -195.49905395507812, + "KL/mean": -272.87152099609375, + "KL/rejected_KL_mean": -350.2440490722656, + "KL/std": 312.8592834472656, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 154.7449951171875, + "fcm_dpo/q_t": 0.4999850392341614, + "grad_norm": 0.002652752446010709, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 1.2726714611053467, + "logits/rejected": 1.2336527109146118, + "logps/chosen": -246.52560424804688, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -426.736083984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 154.7449951171875, + "margin_dpo/margin_std": 322.3939514160156, + "step": 624 + }, + { + "KL/chosen_KL_mean": -213.73834228515625, + "KL/mean": -271.16351318359375, + "KL/rejected_KL_mean": -328.5887145996094, + "KL/std": 293.5513000488281, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 114.85034942626953, + "fcm_dpo/q_t": 0.4999888837337494, + "grad_norm": 0.0026168376207351685, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 1.3009436130523682, + "logits/rejected": 1.2550803422927856, + "logps/chosen": -267.9459533691406, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -413.525390625, + "loss": 1.3862, + "margin_dpo/margin_mean": 114.85035705566406, + "margin_dpo/margin_std": 311.7130126953125, + "step": 625 + }, + { + "KL/chosen_KL_mean": -169.8229522705078, + "KL/mean": -267.14849853515625, + "KL/rejected_KL_mean": -364.47406005859375, + "KL/std": 340.929931640625, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 194.651123046875, + "fcm_dpo/q_t": 0.499981164932251, + "grad_norm": 0.0032611822243779898, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 1.2939801216125488, + "logits/rejected": 1.2345411777496338, + "logps/chosen": -214.88497924804688, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -454.13775634765625, + "loss": 1.3862, + "margin_dpo/margin_mean": 194.651123046875, + "margin_dpo/margin_std": 399.38238525390625, + "step": 626 + }, + { + "KL/chosen_KL_mean": -223.85272216796875, + "KL/mean": -312.509033203125, + "KL/rejected_KL_mean": -401.1653747558594, + "KL/std": 350.2436218261719, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 177.31263732910156, + "fcm_dpo/q_t": 0.4999828338623047, + "grad_norm": 0.0033359085209667683, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 1.2410776615142822, + "logits/rejected": 1.183640718460083, + "logps/chosen": -282.643798828125, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -496.0733947753906, + "loss": 1.3862, + "margin_dpo/margin_mean": 177.3126220703125, + "margin_dpo/margin_std": 358.99542236328125, + "step": 627 + }, + { + "KL/chosen_KL_mean": -220.1020050048828, + "KL/mean": -270.9043273925781, + "KL/rejected_KL_mean": -321.7066650390625, + "KL/std": 269.397705078125, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 101.60467529296875, + "fcm_dpo/q_t": 0.49999016523361206, + "grad_norm": 0.0020546908490359783, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 1.292949914932251, + "logits/rejected": 1.277388572692871, + "logps/chosen": -272.90557861328125, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -398.20135498046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 101.60467529296875, + "margin_dpo/margin_std": 336.76214599609375, + "step": 628 + }, + { + "KL/chosen_KL_mean": -258.51788330078125, + "KL/mean": -296.933349609375, + "KL/rejected_KL_mean": -335.34881591796875, + "KL/std": 292.3125, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 76.83094787597656, + "fcm_dpo/q_t": 0.4999925494194031, + "grad_norm": 0.0015813360223546624, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 1.2685928344726562, + "logits/rejected": 1.2776813507080078, + "logps/chosen": -329.2353820800781, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -414.31158447265625, + "loss": 1.3863, + "margin_dpo/margin_mean": 76.83094787597656, + "margin_dpo/margin_std": 334.86175537109375, + "step": 629 + }, + { + "KL/chosen_KL_mean": -204.28152465820312, + "KL/mean": -270.9383544921875, + "KL/rejected_KL_mean": -337.5951843261719, + "KL/std": 271.621826171875, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 133.31365966796875, + "fcm_dpo/q_t": 0.49998709559440613, + "grad_norm": 0.0021148163359612226, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 1.272836685180664, + "logits/rejected": 1.2661714553833008, + "logps/chosen": -260.4829406738281, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -412.29327392578125, + "loss": 1.3862, + "margin_dpo/margin_mean": 133.31365966796875, + "margin_dpo/margin_std": 350.02862548828125, + "step": 630 + }, + { + "KL/chosen_KL_mean": -239.1774444580078, + "KL/mean": -327.0312805175781, + "KL/rejected_KL_mean": -414.8851318359375, + "KL/std": 334.594482421875, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 175.70767211914062, + "fcm_dpo/q_t": 0.4999829828739166, + "grad_norm": 0.0031651852186769247, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 1.3142149448394775, + "logits/rejected": 1.264453411102295, + "logps/chosen": -297.998046875, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -511.3995056152344, + "loss": 1.3862, + "margin_dpo/margin_mean": 175.70765686035156, + "margin_dpo/margin_std": 397.9634094238281, + "step": 631 + }, + { + "KL/chosen_KL_mean": -214.4776153564453, + "KL/mean": -264.37371826171875, + "KL/rejected_KL_mean": -314.269775390625, + "KL/std": 248.91575622558594, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.79216003417969, + "fcm_dpo/q_t": 0.4999903440475464, + "grad_norm": 0.0018715175101533532, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 1.289548397064209, + "logits/rejected": 1.2905744314193726, + "logps/chosen": -273.263671875, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -381.489013671875, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.79216003417969, + "margin_dpo/margin_std": 307.36798095703125, + "step": 632 + }, + { + "KL/chosen_KL_mean": -178.64141845703125, + "KL/mean": -238.59205627441406, + "KL/rejected_KL_mean": -298.5426940917969, + "KL/std": 253.90550231933594, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 119.90129089355469, + "fcm_dpo/q_t": 0.4999883770942688, + "grad_norm": 0.002127637853845954, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 1.2887210845947266, + "logits/rejected": 1.2725958824157715, + "logps/chosen": -230.77162170410156, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -365.7728576660156, + "loss": 1.3862, + "margin_dpo/margin_mean": 119.90128326416016, + "margin_dpo/margin_std": 286.3564453125, + "step": 633 + }, + { + "KL/chosen_KL_mean": -235.68812561035156, + "KL/mean": -247.49952697753906, + "KL/rejected_KL_mean": -259.3109436035156, + "KL/std": 264.06658935546875, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 23.622833251953125, + "fcm_dpo/q_t": 0.49999770522117615, + "grad_norm": 0.0016156296478584409, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 1.3023512363433838, + "logits/rejected": 1.3197746276855469, + "logps/chosen": -296.66790771484375, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -317.8191833496094, + "loss": 1.3863, + "margin_dpo/margin_mean": 23.62282943725586, + "margin_dpo/margin_std": 295.5430908203125, + "step": 634 + }, + { + "KL/chosen_KL_mean": -270.3648376464844, + "KL/mean": -338.20135498046875, + "KL/rejected_KL_mean": -406.0378723144531, + "KL/std": 269.56549072265625, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 135.67300415039062, + "fcm_dpo/q_t": 0.4999868869781494, + "grad_norm": 0.0024493515957146883, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 1.2402102947235107, + "logits/rejected": 1.2276395559310913, + "logps/chosen": -336.3378601074219, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -491.6510314941406, + "loss": 1.3862, + "margin_dpo/margin_mean": 135.67300415039062, + "margin_dpo/margin_std": 318.53192138671875, + "step": 635 + }, + { + "KL/chosen_KL_mean": -190.3470001220703, + "KL/mean": -266.31427001953125, + "KL/rejected_KL_mean": -342.281494140625, + "KL/std": 254.54660034179688, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 151.93450927734375, + "fcm_dpo/q_t": 0.4999852776527405, + "grad_norm": 0.002561971079558134, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 1.2839725017547607, + "logits/rejected": 1.2400085926055908, + "logps/chosen": -239.48716735839844, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -423.55120849609375, + "loss": 1.3862, + "margin_dpo/margin_mean": 151.93450927734375, + "margin_dpo/margin_std": 261.3985595703125, + "step": 636 + }, + { + "KL/chosen_KL_mean": -266.4497985839844, + "KL/mean": -329.82757568359375, + "KL/rejected_KL_mean": -393.205322265625, + "KL/std": 320.78643798828125, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 126.7555160522461, + "fcm_dpo/q_t": 0.49998772144317627, + "grad_norm": 0.002606100868433714, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 1.2443512678146362, + "logits/rejected": 1.230696201324463, + "logps/chosen": -340.1463928222656, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -476.22021484375, + "loss": 1.3862, + "margin_dpo/margin_mean": 126.75550842285156, + "margin_dpo/margin_std": 355.1866149902344, + "step": 637 + }, + { + "KL/chosen_KL_mean": -243.36972045898438, + "KL/mean": -302.26416015625, + "KL/rejected_KL_mean": -361.1585388183594, + "KL/std": 264.62188720703125, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 117.788818359375, + "fcm_dpo/q_t": 0.4999886155128479, + "grad_norm": 0.0024036893155425787, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 1.290708065032959, + "logits/rejected": 1.2749075889587402, + "logps/chosen": -306.15130615234375, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -446.5633239746094, + "loss": 1.3862, + "margin_dpo/margin_mean": 117.788818359375, + "margin_dpo/margin_std": 344.58038330078125, + "step": 638 + }, + { + "KL/chosen_KL_mean": -215.73281860351562, + "KL/mean": -291.6164855957031, + "KL/rejected_KL_mean": -367.5001220703125, + "KL/std": 315.470947265625, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 151.767333984375, + "fcm_dpo/q_t": 0.49998530745506287, + "grad_norm": 0.002579744439572096, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 1.2794743776321411, + "logits/rejected": 1.251741886138916, + "logps/chosen": -269.4994201660156, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -439.80023193359375, + "loss": 1.3862, + "margin_dpo/margin_mean": 151.767333984375, + "margin_dpo/margin_std": 374.64910888671875, + "step": 639 + }, + { + "KL/chosen_KL_mean": -203.71878051757812, + "KL/mean": -270.4925842285156, + "KL/rejected_KL_mean": -337.2663879394531, + "KL/std": 287.63861083984375, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 133.547607421875, + "fcm_dpo/q_t": 0.49998706579208374, + "grad_norm": 0.0021755900233983994, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 1.3102037906646729, + "logits/rejected": 1.282835602760315, + "logps/chosen": -255.13656616210938, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -414.545166015625, + "loss": 1.3862, + "margin_dpo/margin_mean": 133.54759216308594, + "margin_dpo/margin_std": 318.0604248046875, + "step": 640 + }, + { + "KL/chosen_KL_mean": -253.23318481445312, + "KL/mean": -302.85894775390625, + "KL/rejected_KL_mean": -352.48468017578125, + "KL/std": 273.1927795410156, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 99.2514877319336, + "fcm_dpo/q_t": 0.49999040365219116, + "grad_norm": 0.0019294384401291609, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 1.2644659280776978, + "logits/rejected": 1.2682955265045166, + "logps/chosen": -324.287841796875, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -434.72869873046875, + "loss": 1.3863, + "margin_dpo/margin_mean": 99.2514877319336, + "margin_dpo/margin_std": 286.9643859863281, + "step": 641 + }, + { + "KL/chosen_KL_mean": -276.5420227050781, + "KL/mean": -290.9453430175781, + "KL/rejected_KL_mean": -305.3486633300781, + "KL/std": 272.9807434082031, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 28.806636810302734, + "fcm_dpo/q_t": 0.49999719858169556, + "grad_norm": 0.001907353405840695, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 1.2331547737121582, + "logits/rejected": 1.250314474105835, + "logps/chosen": -345.4713134765625, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -376.20550537109375, + "loss": 1.3863, + "margin_dpo/margin_mean": 28.8066349029541, + "margin_dpo/margin_std": 318.6273193359375, + "step": 642 + }, + { + "KL/chosen_KL_mean": -229.24893188476562, + "KL/mean": -313.97808837890625, + "KL/rejected_KL_mean": -398.70721435546875, + "KL/std": 334.5547790527344, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 169.45823669433594, + "fcm_dpo/q_t": 0.4999835789203644, + "grad_norm": 0.003335776040330529, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 1.2515963315963745, + "logits/rejected": 1.2289414405822754, + "logps/chosen": -294.5579833984375, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -482.3233337402344, + "loss": 1.3862, + "margin_dpo/margin_mean": 169.458251953125, + "margin_dpo/margin_std": 398.0389099121094, + "step": 643 + }, + { + "KL/chosen_KL_mean": -195.06884765625, + "KL/mean": -232.06622314453125, + "KL/rejected_KL_mean": -269.0635986328125, + "KL/std": 229.22683715820312, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 73.99472045898438, + "fcm_dpo/q_t": 0.49999281764030457, + "grad_norm": 0.0016697602113708854, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 1.319830060005188, + "logits/rejected": 1.3086210489273071, + "logps/chosen": -246.0714569091797, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -333.5273132324219, + "loss": 1.3863, + "margin_dpo/margin_mean": 73.99470520019531, + "margin_dpo/margin_std": 218.09716796875, + "step": 644 + }, + { + "KL/chosen_KL_mean": -238.21824645996094, + "KL/mean": -267.15887451171875, + "KL/rejected_KL_mean": -296.0995178222656, + "KL/std": 247.69064331054688, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 57.881282806396484, + "fcm_dpo/q_t": 0.4999943971633911, + "grad_norm": 0.001644918229430914, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 1.2519395351409912, + "logits/rejected": 1.2480509281158447, + "logps/chosen": -299.181640625, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -365.83306884765625, + "loss": 1.3863, + "margin_dpo/margin_mean": 57.88128662109375, + "margin_dpo/margin_std": 300.2716064453125, + "step": 645 + }, + { + "KL/chosen_KL_mean": -262.1314697265625, + "KL/mean": -314.58929443359375, + "KL/rejected_KL_mean": -367.047119140625, + "KL/std": 342.90087890625, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 104.91563415527344, + "fcm_dpo/q_t": 0.4999898374080658, + "grad_norm": 0.0021173881832510233, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 1.2763549089431763, + "logits/rejected": 1.2538020610809326, + "logps/chosen": -324.42156982421875, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -452.59521484375, + "loss": 1.3863, + "margin_dpo/margin_mean": 104.91563415527344, + "margin_dpo/margin_std": 296.7239990234375, + "step": 646 + }, + { + "KL/chosen_KL_mean": -277.2899475097656, + "KL/mean": -347.98150634765625, + "KL/rejected_KL_mean": -418.67303466796875, + "KL/std": 369.2318115234375, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 141.3831024169922, + "fcm_dpo/q_t": 0.49998632073402405, + "grad_norm": 0.002953104441985488, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 1.2947700023651123, + "logits/rejected": 1.2425816059112549, + "logps/chosen": -344.8058166503906, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -520.1817626953125, + "loss": 1.3862, + "margin_dpo/margin_mean": 141.3831024169922, + "margin_dpo/margin_std": 402.0182189941406, + "step": 647 + }, + { + "KL/chosen_KL_mean": -248.36538696289062, + "KL/mean": -310.21771240234375, + "KL/rejected_KL_mean": -372.0700378417969, + "KL/std": 275.6451416015625, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 123.70465087890625, + "fcm_dpo/q_t": 0.49998801946640015, + "grad_norm": 0.002356282202526927, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 1.2679743766784668, + "logits/rejected": 1.2473814487457275, + "logps/chosen": -312.9613037109375, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -455.4540710449219, + "loss": 1.3862, + "margin_dpo/margin_mean": 123.70465087890625, + "margin_dpo/margin_std": 370.0848693847656, + "step": 648 + }, + { + "KL/chosen_KL_mean": -202.95803833007812, + "KL/mean": -254.43443298339844, + "KL/rejected_KL_mean": -305.91082763671875, + "KL/std": 240.42996215820312, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.95277404785156, + "fcm_dpo/q_t": 0.4999900460243225, + "grad_norm": 0.0025812452659010887, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 1.308830738067627, + "logits/rejected": 1.2813698053359985, + "logps/chosen": -252.2677001953125, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -379.64794921875, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.95277404785156, + "margin_dpo/margin_std": 293.2994384765625, + "step": 649 + }, + { + "KL/chosen_KL_mean": -175.028076171875, + "KL/mean": -253.37965393066406, + "KL/rejected_KL_mean": -331.731201171875, + "KL/std": 274.1990661621094, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 156.70315551757812, + "fcm_dpo/q_t": 0.49998483061790466, + "grad_norm": 0.002816020045429468, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 1.3074793815612793, + "logits/rejected": 1.2863340377807617, + "logps/chosen": -230.09133911132812, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -409.1273193359375, + "loss": 1.3862, + "margin_dpo/margin_mean": 156.70314025878906, + "margin_dpo/margin_std": 366.322998046875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -214.81884765625, + "KL/mean": -288.9599609375, + "KL/rejected_KL_mean": -363.10113525390625, + "KL/std": 292.44451904296875, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 148.2822723388672, + "fcm_dpo/q_t": 0.49998563528060913, + "grad_norm": 0.002445911755785346, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 1.288893461227417, + "logits/rejected": 1.2606956958770752, + "logps/chosen": -268.8840026855469, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -440.89190673828125, + "loss": 1.3862, + "margin_dpo/margin_mean": 148.2822723388672, + "margin_dpo/margin_std": 326.09967041015625, + "step": 651 + }, + { + "KL/chosen_KL_mean": -275.9385986328125, + "KL/mean": -314.86370849609375, + "KL/rejected_KL_mean": -353.788818359375, + "KL/std": 326.7114562988281, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 77.85020446777344, + "fcm_dpo/q_t": 0.4999924600124359, + "grad_norm": 0.0021558792795985937, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 1.2859222888946533, + "logits/rejected": 1.265965223312378, + "logps/chosen": -339.57891845703125, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -432.6576232910156, + "loss": 1.3863, + "margin_dpo/margin_mean": 77.85020446777344, + "margin_dpo/margin_std": 423.6152648925781, + "step": 652 + }, + { + "KL/chosen_KL_mean": -228.88973999023438, + "KL/mean": -280.29022216796875, + "KL/rejected_KL_mean": -331.6907043457031, + "KL/std": 262.5857849121094, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 102.80097961425781, + "fcm_dpo/q_t": 0.4999900460243225, + "grad_norm": 0.001885903999209404, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 1.2586016654968262, + "logits/rejected": 1.2437849044799805, + "logps/chosen": -290.55810546875, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -405.52081298828125, + "loss": 1.3863, + "margin_dpo/margin_mean": 102.80097961425781, + "margin_dpo/margin_std": 331.70751953125, + "step": 653 + }, + { + "KL/chosen_KL_mean": -235.8380126953125, + "KL/mean": -312.2591552734375, + "KL/rejected_KL_mean": -388.6803283691406, + "KL/std": 327.74395751953125, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 152.84230041503906, + "fcm_dpo/q_t": 0.4999852180480957, + "grad_norm": 0.002882143948227167, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 1.2653387784957886, + "logits/rejected": 1.2258384227752686, + "logps/chosen": -293.4062805175781, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -476.42822265625, + "loss": 1.3862, + "margin_dpo/margin_mean": 152.84230041503906, + "margin_dpo/margin_std": 319.22900390625, + "step": 654 + }, + { + "KL/chosen_KL_mean": -178.378173828125, + "KL/mean": -272.81280517578125, + "KL/rejected_KL_mean": -367.2474670410156, + "KL/std": 287.7238464355469, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 188.86932373046875, + "fcm_dpo/q_t": 0.49998173117637634, + "grad_norm": 0.003305543912574649, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 1.2574398517608643, + "logits/rejected": 1.2300739288330078, + "logps/chosen": -230.5253143310547, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -448.09759521484375, + "loss": 1.3862, + "margin_dpo/margin_mean": 188.86932373046875, + "margin_dpo/margin_std": 355.13861083984375, + "step": 655 + }, + { + "KL/chosen_KL_mean": -268.1697692871094, + "KL/mean": -328.5416564941406, + "KL/rejected_KL_mean": -388.91357421875, + "KL/std": 299.40435791015625, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 120.74380493164062, + "fcm_dpo/q_t": 0.499988317489624, + "grad_norm": 0.0024166665971279144, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 1.2543549537658691, + "logits/rejected": 1.2443668842315674, + "logps/chosen": -341.4264831542969, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -474.26483154296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 120.74380493164062, + "margin_dpo/margin_std": 330.2796630859375, + "step": 656 + }, + { + "KL/chosen_KL_mean": -185.50474548339844, + "KL/mean": -276.64654541015625, + "KL/rejected_KL_mean": -367.788330078125, + "KL/std": 325.72955322265625, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.28355407714844, + "fcm_dpo/q_t": 0.4999823570251465, + "grad_norm": 0.00303459819406271, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 1.3113363981246948, + "logits/rejected": 1.2885608673095703, + "logps/chosen": -235.2281494140625, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -442.9451904296875, + "loss": 1.3862, + "margin_dpo/margin_mean": 182.28355407714844, + "margin_dpo/margin_std": 396.7044677734375, + "step": 657 + }, + { + "KL/chosen_KL_mean": -248.78866577148438, + "KL/mean": -298.27093505859375, + "KL/rejected_KL_mean": -347.7532043457031, + "KL/std": 330.69586181640625, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 98.96455383300781, + "fcm_dpo/q_t": 0.49999043345451355, + "grad_norm": 0.0021189674735069275, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 1.2897260189056396, + "logits/rejected": 1.271322250366211, + "logps/chosen": -311.83502197265625, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -431.2028503417969, + "loss": 1.3863, + "margin_dpo/margin_mean": 98.96455383300781, + "margin_dpo/margin_std": 376.11456298828125, + "step": 658 + }, + { + "KL/chosen_KL_mean": -244.0128936767578, + "KL/mean": -291.29986572265625, + "KL/rejected_KL_mean": -338.5868225097656, + "KL/std": 314.4420166015625, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 94.57395935058594, + "fcm_dpo/q_t": 0.4999908208847046, + "grad_norm": 0.0024397147353738546, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 1.3078527450561523, + "logits/rejected": 1.281451940536499, + "logps/chosen": -299.09307861328125, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -410.497314453125, + "loss": 1.3863, + "margin_dpo/margin_mean": 94.5739517211914, + "margin_dpo/margin_std": 354.7186279296875, + "step": 659 + }, + { + "KL/chosen_KL_mean": -202.13861083984375, + "KL/mean": -280.438720703125, + "KL/rejected_KL_mean": -358.7388610839844, + "KL/std": 307.3833923339844, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 156.60025024414062, + "fcm_dpo/q_t": 0.49998486042022705, + "grad_norm": 0.002643043175339699, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 1.329068660736084, + "logits/rejected": 1.3038125038146973, + "logps/chosen": -256.6645202636719, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -439.97491455078125, + "loss": 1.3862, + "margin_dpo/margin_mean": 156.60025024414062, + "margin_dpo/margin_std": 385.9468688964844, + "step": 660 + }, + { + "KL/chosen_KL_mean": -255.35760498046875, + "KL/mean": -313.8293762207031, + "KL/rejected_KL_mean": -372.3011474609375, + "KL/std": 311.5862121582031, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 3.8711289107595803e-07, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 116.94351959228516, + "fcm_dpo/q_t": 0.4999886751174927, + "grad_norm": 0.0025908024981617928, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 1.2794947624206543, + "logits/rejected": 1.2472293376922607, + "logps/chosen": -315.7302551269531, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -449.7298583984375, + "loss": 1.3862, + "margin_dpo/margin_mean": 116.94351959228516, + "margin_dpo/margin_std": 371.20477294921875, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.370076598991244, + "train_runtime": 1650.2951, + "train_samples_per_second": 25.654, + "train_steps_per_second": 0.401 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}