From 6d0881812d8b77ab7fa6eeaa803b659f206a0cc0 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 26 May 2026 12:58:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: jackf857/llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4 Source: Original Platform --- .gitattributes | 36 + README.md | 81 + all_results.json | 27 + config.json | 29 + eval_results.json | 21 + generation_config.json | 9 + model-00001-of-00006.safetensors | 3 + model-00001-of-00007.safetensors | 3 + model-00002-of-00006.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00006.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00006.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00006.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00006.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++++ train_results.json | 9 + trainer_state.json | 11056 +++++++++++++++++++++++++++++ training_args.bin | 3 + 26 files changed, 13698 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 eval_results.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00006.safetensors create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00006.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00006.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00006.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00006.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00006.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..8d7eeac --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-ultrachat-8xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- HuggingFaceH4/ultrafeedback_binarized +model-index: +- name: llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-ultrafeedback-4xh200-batch-128-q_t-0.43-s_star-0.4 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-ultrachat-8xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-ultrachat-8xh200) on the HuggingFaceH4/ultrafeedback_binarized dataset. +It achieves the following results on the evaluation set: +- Loss: 0.5983 +- Fcm Dpo/beta: 0.0027 +- Margin Dpo/margin Mean: 105.4011 +- Margin Dpo/margin Std: 182.7466 +- Logps/chosen: -546.8945 +- Logps/rejected: -631.3989 +- Logps/ref Chosen: -287.8268 +- Logps/ref Rejected: -266.9300 +- Kl/chosen Kl Mean: -259.0677 +- Kl/rejected Kl Mean: -364.4689 +- Kl/mean: -311.7683 +- Kl/std: 162.3682 +- Logits/chosen: -0.8539 +- Logits/rejected: -0.8364 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 4 +- eval_batch_size: 2 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 128 +- total_eval_batch_size: 8 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Fcm Dpo/beta | Margin Dpo/margin Mean | Margin Dpo/margin Std | Logps/chosen | Logps/rejected | Logps/ref Chosen | Logps/ref Rejected | Kl/chosen Kl Mean | Kl/rejected Kl Mean | Kl/mean | Kl/std | Logits/chosen | Logits/rejected | +|:-------------:|:------:|:----:|:---------------:|:------------:|:----------------------:|:---------------------:|:------------:|:--------------:|:----------------:|:------------------:|:-----------------:|:-------------------:|:---------:|:--------:|:-------------:|:---------------:| +| 4.9173 | 0.4188 | 200 | 0.5938 | 0.0054 | 55.2587 | 96.2979 | -406.3043 | -440.6663 | -287.8268 | -266.9300 | -118.4775 | -173.7362 | -146.1068 | 87.6889 | -0.9034 | -0.8850 | +| 4.8145 | 0.8377 | 400 | 0.5983 | 0.0027 | 105.4011 | 182.7466 | -546.8945 | -631.3989 | -287.8268 | -266.9300 | -259.0677 | -364.4689 | -311.7683 | 162.3682 | -0.8539 | -0.8364 | + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..6452f42 --- /dev/null +++ b/all_results.json @@ -0,0 +1,27 @@ +{ + "epoch": 0.9989528795811519, + "eval_KL/chosen_KL_mean": -262.6319580078125, + "eval_KL/mean": -316.4208679199219, + "eval_KL/rejected_KL_mean": -370.2097473144531, + "eval_KL/std": 165.86854553222656, + "eval_fcm_dpo/beta": 0.0024308497086167336, + "eval_logits/chosen": -0.846673846244812, + "eval_logits/rejected": -0.8289116621017456, + "eval_logps/chosen": -550.458740234375, + "eval_logps/ref_chosen": -287.8268127441406, + "eval_logps/ref_rejected": -266.9300231933594, + "eval_logps/rejected": -637.1398315429688, + "eval_loss": 0.6021535396575928, + "eval_margin_dpo/margin_mean": 107.57779693603516, + "eval_margin_dpo/margin_std": 186.7196807861328, + "eval_runtime": 78.3489, + "eval_samples": 2000, + "eval_samples_per_second": 25.527, + "eval_steps_per_second": 3.191, + "total_flos": 0.0, + "train_loss": 4.88172167052263, + "train_runtime": 7274.147, + "train_samples": 61135, + "train_samples_per_second": 8.404, + "train_steps_per_second": 0.066 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/eval_results.json b/eval_results.json new file mode 100644 index 0000000..b1f6fe6 --- /dev/null +++ b/eval_results.json @@ -0,0 +1,21 @@ +{ + "epoch": 0.9989528795811519, + "eval_KL/chosen_KL_mean": -262.6319580078125, + "eval_KL/mean": -316.4208679199219, + "eval_KL/rejected_KL_mean": -370.2097473144531, + "eval_KL/std": 165.86854553222656, + "eval_fcm_dpo/beta": 0.0024308497086167336, + "eval_logits/chosen": -0.846673846244812, + "eval_logits/rejected": -0.8289116621017456, + "eval_logps/chosen": -550.458740234375, + "eval_logps/ref_chosen": -287.8268127441406, + "eval_logps/ref_rejected": -266.9300231933594, + "eval_logps/rejected": -637.1398315429688, + "eval_loss": 0.6021535396575928, + "eval_margin_dpo/margin_mean": 107.57779693603516, + "eval_margin_dpo/margin_std": 186.7196807861328, + "eval_runtime": 78.3489, + "eval_samples": 2000, + "eval_samples_per_second": 25.527, + "eval_steps_per_second": 3.191 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00006.safetensors b/model-00001-of-00006.safetensors new file mode 100644 index 0000000..77b1db1 --- /dev/null +++ b/model-00001-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7d9493f5da31e9397a115e15b9d3f8b14b13c1954f8527900b4872d78376ba3 +size 4943127280 diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..53c91b9 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60ffb16f2671251b42e255ebf6b8b7ea22ceb78cb8e81c4d1f4f472a3d1c84e +size 4886466168 diff --git a/model-00002-of-00006.safetensors b/model-00002-of-00006.safetensors new file mode 100644 index 0000000..c95b6cd --- /dev/null +++ b/model-00002-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3319d250cc3691ad17cb0cced45b8e4044e837606ee7500d741ebb76273ac97d +size 4999813072 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..14390ed --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9beb05b7808731189c6084886502453bb786a0f42e67c8b7549600b7597a12b2 +size 4832007448 diff --git a/model-00003-of-00006.safetensors b/model-00003-of-00006.safetensors new file mode 100644 index 0000000..807ccd5 --- /dev/null +++ b/model-00003-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d689980f71aea2acf92307f5f84d526d8f982c23e8bce85af65f4ef71818b356 +size 4832007496 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..8cef85e --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c492f5c5087c94971d17270b6bfcf76f9d4eee3533cf48a5f03ea1129afaafd +size 4999813112 diff --git a/model-00004-of-00006.safetensors b/model-00004-of-00006.safetensors new file mode 100644 index 0000000..b756586 --- /dev/null +++ b/model-00004-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:400f7973afb8f68eb35465c83c9937a167906bec0b99bf245ce4306718050fc1 +size 4999813120 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..35acbe3 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8299b8bd4ce63ff69c4d9bd0ef4dd2575be88453ca43a626195737088478e76e +size 4999813128 diff --git a/model-00005-of-00006.safetensors b/model-00005-of-00006.safetensors new file mode 100644 index 0000000..80774f0 --- /dev/null +++ b/model-00005-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704af675f4680a581ccd60a5e3f2d1c39c86a503a3f9bbbb46b14b75bdef06ef +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..4512d1d --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb213d81f21be411876c29284326e5794017f98818960f32dfb19a5526c54523 +size 4832007496 diff --git a/model-00006-of-00006.safetensors b/model-00006-of-00006.safetensors new file mode 100644 index 0000000..f8d9072 --- /dev/null +++ b/model-00006-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc3dae183276a14653bf7ac2f5fb1cc78f66b3a55e674ab5999ab438239baaf +size 4194472744 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..fb1cd20 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92655f73697afce5304b211dadff5f18912c5cbd1289045d0f1f2d437edb005f +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..a5da843 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc5fdc9011c9a8c64c1fcb26e922ef7b142b50fb1bab458379fc8c0bb73c5dcb +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..dd44caa --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.9989528795811519, + "total_flos": 0.0, + "train_loss": 4.88172167052263, + "train_runtime": 7274.147, + "train_samples": 61135, + "train_samples_per_second": 8.404, + "train_steps_per_second": 0.066 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c2507cf --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,11056 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9989528795811519, + "eval_steps": 200, + "global_step": 477, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": -0.05443763732910156, + "KL/mean": -0.04046645760536194, + "KL/rejected_KL_mean": -0.026498794555664062, + "KL/std": 0.47091516852378845, + "epoch": 0.0020942408376963353, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02793481945991516, + "fcm_dpo/q_t": 0.500069797039032, + "grad_norm": 28.589929580688477, + "learning_rate": 0.0, + "logits/chosen": -0.5898098945617676, + "logits/rejected": -0.604260265827179, + "logps/chosen": -275.28570556640625, + "logps/ref_chosen": -275.2312927246094, + "logps/ref_rejected": -222.9380340576172, + "logps/rejected": -222.96453857421875, + "loss": 5.5463, + "margin_dpo/margin_mean": -0.02793477475643158, + "margin_dpo/margin_std": 0.5724214911460876, + "step": 1 + }, + { + "KL/chosen_KL_mean": 0.04458808898925781, + "KL/mean": 0.0374295711517334, + "KL/rejected_KL_mean": 0.030269622802734375, + "KL/std": 0.49036121368408203, + "epoch": 0.004188481675392671, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.014312177896499634, + "fcm_dpo/q_t": 0.4999642074108124, + "grad_norm": 27.881349563598633, + "learning_rate": 1.0416666666666666e-08, + "logits/chosen": -0.6574729681015015, + "logits/rejected": -0.6464410424232483, + "logps/chosen": -264.7165222167969, + "logps/ref_chosen": -264.7611083984375, + "logps/ref_rejected": -242.5597686767578, + "logps/rejected": -242.52951049804688, + "loss": 5.5446, + "margin_dpo/margin_mean": 0.014312252402305603, + "margin_dpo/margin_std": 0.6423971652984619, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.015283584594726562, + "KL/mean": -0.027470149099826813, + "KL/rejected_KL_mean": -0.03966331481933594, + "KL/std": 0.4722851514816284, + "epoch": 0.0062827225130890054, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02437432110309601, + "fcm_dpo/q_t": 0.49993908405303955, + "grad_norm": 25.772430419921875, + "learning_rate": 2.083333333333333e-08, + "logits/chosen": -0.6839795112609863, + "logits/rejected": -0.7351416349411011, + "logps/chosen": -274.1171875, + "logps/ref_chosen": -274.1018981933594, + "logps/ref_rejected": -286.5882568359375, + "logps/rejected": -286.6279296875, + "loss": 5.5442, + "margin_dpo/margin_mean": 0.024372458457946777, + "margin_dpo/margin_std": 0.6266783475875854, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.04073333740234375, + "KL/mean": -0.02466951310634613, + "KL/rejected_KL_mean": -0.0900726318359375, + "KL/std": 0.5628562569618225, + "epoch": 0.008376963350785341, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13081058859825134, + "fcm_dpo/q_t": 0.49967294931411743, + "grad_norm": 31.76841926574707, + "learning_rate": 3.125e-08, + "logits/chosen": -0.6164096593856812, + "logits/rejected": -0.61078280210495, + "logps/chosen": -329.7975158691406, + "logps/ref_chosen": -329.8382568359375, + "logps/ref_rejected": -303.2850646972656, + "logps/rejected": -303.3750915527344, + "loss": 5.54, + "margin_dpo/margin_mean": 0.1308116316795349, + "margin_dpo/margin_std": 0.7712123990058899, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.004119873046875, + "KL/mean": 0.020755663514137268, + "KL/rejected_KL_mean": 0.045635223388671875, + "KL/std": 0.533147931098938, + "epoch": 0.010471204188481676, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04974430799484253, + "fcm_dpo/q_t": 0.5001243352890015, + "grad_norm": 29.5401554107666, + "learning_rate": 4.166666666666666e-08, + "logits/chosen": -0.5749531984329224, + "logits/rejected": -0.5917044281959534, + "logps/chosen": -301.7430419921875, + "logps/ref_chosen": -301.7389221191406, + "logps/ref_rejected": -274.7654724121094, + "logps/rejected": -274.7198181152344, + "loss": 5.5472, + "margin_dpo/margin_mean": -0.049745798110961914, + "margin_dpo/margin_std": 0.8052065968513489, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.016300201416015625, + "KL/mean": 0.0033769533038139343, + "KL/rejected_KL_mean": 0.023052215576171875, + "KL/std": 0.5462159514427185, + "epoch": 0.012565445026178011, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03935401141643524, + "fcm_dpo/q_t": 0.5000984072685242, + "grad_norm": 28.32386589050293, + "learning_rate": 5.208333333333333e-08, + "logits/chosen": -0.6760701537132263, + "logits/rejected": -0.6386649012565613, + "logps/chosen": -285.7109680175781, + "logps/ref_chosen": -285.6946716308594, + "logps/ref_rejected": -245.8200225830078, + "logps/rejected": -245.79696655273438, + "loss": 5.5468, + "margin_dpo/margin_mean": -0.03935439884662628, + "margin_dpo/margin_std": 0.7249687314033508, + "step": 6 + }, + { + "KL/chosen_KL_mean": -0.08362960815429688, + "KL/mean": -0.03842785954475403, + "KL/rejected_KL_mean": 0.006763458251953125, + "KL/std": 0.4993395209312439, + "epoch": 0.014659685863874346, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.09039899706840515, + "fcm_dpo/q_t": 0.5002260208129883, + "grad_norm": 28.561147689819336, + "learning_rate": 6.25e-08, + "logits/chosen": -0.5755224823951721, + "logits/rejected": -0.6081489324569702, + "logps/chosen": -264.7390441894531, + "logps/ref_chosen": -264.65545654296875, + "logps/ref_rejected": -253.10305786132812, + "logps/rejected": -253.09628295898438, + "loss": 5.5488, + "margin_dpo/margin_mean": -0.09039971232414246, + "margin_dpo/margin_std": 0.6376783847808838, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.041156768798828125, + "KL/mean": -0.013764090836048126, + "KL/rejected_KL_mean": -0.06868171691894531, + "KL/std": 0.5888418555259705, + "epoch": 0.016753926701570682, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10983617603778839, + "fcm_dpo/q_t": 0.49972546100616455, + "grad_norm": 30.76198387145996, + "learning_rate": 7.291666666666667e-08, + "logits/chosen": -0.6753714680671692, + "logits/rejected": -0.681854784488678, + "logps/chosen": -354.14764404296875, + "logps/ref_chosen": -354.1887512207031, + "logps/ref_rejected": -282.9112243652344, + "logps/rejected": -282.9798889160156, + "loss": 5.5409, + "margin_dpo/margin_mean": 0.10983628034591675, + "margin_dpo/margin_std": 0.7891420125961304, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.0035858154296875, + "KL/mean": 0.005865432322025299, + "KL/rejected_KL_mean": 0.008136749267578125, + "KL/std": 0.5102388858795166, + "epoch": 0.018848167539267015, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.004547819495201111, + "fcm_dpo/q_t": 0.5000113844871521, + "grad_norm": 27.88920021057129, + "learning_rate": 8.333333333333333e-08, + "logits/chosen": -0.6402922868728638, + "logits/rejected": -0.6586679220199585, + "logps/chosen": -285.546630859375, + "logps/ref_chosen": -285.5502014160156, + "logps/ref_rejected": -267.99664306640625, + "logps/rejected": -267.988525390625, + "loss": 5.5454, + "margin_dpo/margin_mean": -0.004548460245132446, + "margin_dpo/margin_std": 0.7229992747306824, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.06154441833496094, + "KL/mean": 0.019467219710350037, + "KL/rejected_KL_mean": -0.022617340087890625, + "KL/std": 0.4935854971408844, + "epoch": 0.020942408376963352, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08416536450386047, + "fcm_dpo/q_t": 0.49978959560394287, + "grad_norm": 26.521408081054688, + "learning_rate": 9.375e-08, + "logits/chosen": -0.6825000047683716, + "logits/rejected": -0.6766196489334106, + "logps/chosen": -251.850830078125, + "logps/ref_chosen": -251.91238403320312, + "logps/ref_rejected": -226.45260620117188, + "logps/rejected": -226.4752197265625, + "loss": 5.5419, + "margin_dpo/margin_mean": 0.08416460454463959, + "margin_dpo/margin_std": 0.7020270228385925, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.02030181884765625, + "KL/mean": 0.06047465652227402, + "KL/rejected_KL_mean": 0.10064697265625, + "KL/std": 0.5014757513999939, + "epoch": 0.023036649214659685, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.08033938705921173, + "fcm_dpo/q_t": 0.5002008080482483, + "grad_norm": 28.9848575592041, + "learning_rate": 1.0416666666666667e-07, + "logits/chosen": -0.5878165364265442, + "logits/rejected": -0.6417873501777649, + "logps/chosen": -301.06317138671875, + "logps/ref_chosen": -301.08343505859375, + "logps/ref_rejected": -259.546630859375, + "logps/rejected": -259.44598388671875, + "loss": 5.5484, + "margin_dpo/margin_mean": -0.08033882081508636, + "margin_dpo/margin_std": 0.6635822653770447, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.0308685302734375, + "KL/mean": -0.08398044854402542, + "KL/rejected_KL_mean": -0.1370849609375, + "KL/std": 0.5727797746658325, + "epoch": 0.025130890052356022, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1062176376581192, + "fcm_dpo/q_t": 0.4997345209121704, + "grad_norm": 30.093599319458008, + "learning_rate": 1.1458333333333332e-07, + "logits/chosen": -0.5787484645843506, + "logits/rejected": -0.538584291934967, + "logps/chosen": -287.5789794921875, + "logps/ref_chosen": -287.548095703125, + "logps/ref_rejected": -277.37945556640625, + "logps/rejected": -277.51654052734375, + "loss": 5.541, + "margin_dpo/margin_mean": 0.10621786117553711, + "margin_dpo/margin_std": 0.8124967813491821, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.04111671447753906, + "KL/mean": -0.03685136139392853, + "KL/rejected_KL_mean": -0.11482620239257812, + "KL/std": 0.49719753861427307, + "epoch": 0.027225130890052355, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15594592690467834, + "fcm_dpo/q_t": 0.49961015582084656, + "grad_norm": 27.183223724365234, + "learning_rate": 1.25e-07, + "logits/chosen": -0.6483482122421265, + "logits/rejected": -0.6553612947463989, + "logps/chosen": -270.62530517578125, + "logps/ref_chosen": -270.6664123535156, + "logps/ref_rejected": -274.6546936035156, + "logps/rejected": -274.76953125, + "loss": 5.539, + "margin_dpo/margin_mean": 0.15594591200351715, + "margin_dpo/margin_std": 0.6842634677886963, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.08417701721191406, + "KL/mean": 0.03670407831668854, + "KL/rejected_KL_mean": -0.010768890380859375, + "KL/std": 0.5663655400276184, + "epoch": 0.02931937172774869, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09495711326599121, + "fcm_dpo/q_t": 0.4997625946998596, + "grad_norm": 28.36115264892578, + "learning_rate": 1.3541666666666666e-07, + "logits/chosen": -0.6208177208900452, + "logits/rejected": -0.6492733955383301, + "logps/chosen": -281.5090637207031, + "logps/ref_chosen": -281.59320068359375, + "logps/ref_rejected": -263.52215576171875, + "logps/rejected": -263.532958984375, + "loss": 5.5414, + "margin_dpo/margin_mean": 0.09495654702186584, + "margin_dpo/margin_std": 0.7788461446762085, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.000659942626953125, + "KL/mean": 0.006451621651649475, + "KL/rejected_KL_mean": 0.013570785522460938, + "KL/std": 0.5095148086547852, + "epoch": 0.031413612565445025, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.014226287603378296, + "fcm_dpo/q_t": 0.5000355839729309, + "grad_norm": 30.425567626953125, + "learning_rate": 1.4583333333333335e-07, + "logits/chosen": -0.6453888416290283, + "logits/rejected": -0.6586918830871582, + "logps/chosen": -298.4541015625, + "logps/ref_chosen": -298.45343017578125, + "logps/ref_rejected": -227.17832946777344, + "logps/rejected": -227.16476440429688, + "loss": 5.5458, + "margin_dpo/margin_mean": -0.014227613806724548, + "margin_dpo/margin_std": 0.7101153135299683, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.062591552734375, + "KL/mean": 0.027167625725269318, + "KL/rejected_KL_mean": -0.008253097534179688, + "KL/std": 0.5141140222549438, + "epoch": 0.033507853403141365, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07084225118160248, + "fcm_dpo/q_t": 0.4998228847980499, + "grad_norm": 30.161983489990234, + "learning_rate": 1.5624999999999999e-07, + "logits/chosen": -0.6102164387702942, + "logits/rejected": -0.60810387134552, + "logps/chosen": -293.9040222167969, + "logps/ref_chosen": -293.96661376953125, + "logps/ref_rejected": -250.78443908691406, + "logps/rejected": -250.79269409179688, + "loss": 5.5424, + "margin_dpo/margin_mean": 0.07084193825721741, + "margin_dpo/margin_std": 0.7234665751457214, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.08296775817871094, + "KL/mean": 0.07325167208909988, + "KL/rejected_KL_mean": 0.06353187561035156, + "KL/std": 0.48913800716400146, + "epoch": 0.0356020942408377, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.019437596201896667, + "fcm_dpo/q_t": 0.49995142221450806, + "grad_norm": 27.688617706298828, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5614761114120483, + "logits/rejected": -0.5870969891548157, + "logps/chosen": -262.3110656738281, + "logps/ref_chosen": -262.39398193359375, + "logps/ref_rejected": -248.500244140625, + "logps/rejected": -248.43670654296875, + "loss": 5.5444, + "margin_dpo/margin_mean": 0.019437864422798157, + "margin_dpo/margin_std": 0.6257155537605286, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.00669097900390625, + "KL/mean": -0.022606775164604187, + "KL/rejected_KL_mean": -0.038524627685546875, + "KL/std": 0.49792757630348206, + "epoch": 0.03769633507853403, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03182762861251831, + "fcm_dpo/q_t": 0.49992042779922485, + "grad_norm": 29.69187355041504, + "learning_rate": 1.7708333333333334e-07, + "logits/chosen": -0.6175638437271118, + "logits/rejected": -0.6237700581550598, + "logps/chosen": -293.7159423828125, + "logps/ref_chosen": -293.709228515625, + "logps/ref_rejected": -274.5875244140625, + "logps/rejected": -274.6260681152344, + "loss": 5.544, + "margin_dpo/margin_mean": 0.03182883560657501, + "margin_dpo/margin_std": 0.6748430728912354, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.03246307373046875, + "KL/mean": 0.049572668969631195, + "KL/rejected_KL_mean": 0.06668472290039062, + "KL/std": 0.5367411375045776, + "epoch": 0.039790575916230364, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.034227266907691956, + "fcm_dpo/q_t": 0.5000855326652527, + "grad_norm": 28.177858352661133, + "learning_rate": 1.875e-07, + "logits/chosen": -0.6283941268920898, + "logits/rejected": -0.6246621608734131, + "logps/chosen": -280.23321533203125, + "logps/ref_chosen": -280.26568603515625, + "logps/ref_rejected": -259.9742736816406, + "logps/rejected": -259.9075927734375, + "loss": 5.5466, + "margin_dpo/margin_mean": -0.03422766923904419, + "margin_dpo/margin_std": 0.7041956782341003, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.17637062072753906, + "KL/mean": 0.1533583253622055, + "KL/rejected_KL_mean": 0.1303424835205078, + "KL/std": 0.54378741979599, + "epoch": 0.041884816753926704, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04602688550949097, + "fcm_dpo/q_t": 0.4998849332332611, + "grad_norm": 29.69119644165039, + "learning_rate": 1.9791666666666664e-07, + "logits/chosen": -0.6134122610092163, + "logits/rejected": -0.6440813541412354, + "logps/chosen": -303.71905517578125, + "logps/ref_chosen": -303.8954162597656, + "logps/ref_rejected": -260.214599609375, + "logps/rejected": -260.0842590332031, + "loss": 5.5434, + "margin_dpo/margin_mean": 0.04602590203285217, + "margin_dpo/margin_std": 0.8291985392570496, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.09269332885742188, + "KL/mean": 0.03890705853700638, + "KL/rejected_KL_mean": -0.01488494873046875, + "KL/std": 0.5381225347518921, + "epoch": 0.04397905759162304, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10758335888385773, + "fcm_dpo/q_t": 0.49973100423812866, + "grad_norm": 34.960567474365234, + "learning_rate": 2.0833333333333333e-07, + "logits/chosen": -0.6435227394104004, + "logits/rejected": -0.671474277973175, + "logps/chosen": -301.4407958984375, + "logps/ref_chosen": -301.5334777832031, + "logps/ref_rejected": -280.28900146484375, + "logps/rejected": -280.3038635253906, + "loss": 5.5409, + "margin_dpo/margin_mean": 0.10758353769779205, + "margin_dpo/margin_std": 0.7066560387611389, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.023479461669921875, + "KL/mean": 0.034437984228134155, + "KL/rejected_KL_mean": 0.04539680480957031, + "KL/std": 0.5244945287704468, + "epoch": 0.04607329842931937, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0219249427318573, + "fcm_dpo/q_t": 0.5000548362731934, + "grad_norm": 25.298784255981445, + "learning_rate": 2.1875e-07, + "logits/chosen": -0.6568877696990967, + "logits/rejected": -0.658561110496521, + "logps/chosen": -259.9716491699219, + "logps/ref_chosen": -259.9951477050781, + "logps/ref_rejected": -243.0721435546875, + "logps/rejected": -243.02676391601562, + "loss": 5.5461, + "margin_dpo/margin_mean": -0.021924346685409546, + "margin_dpo/margin_std": 0.713553249835968, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.09302520751953125, + "KL/mean": 0.048005327582359314, + "KL/rejected_KL_mean": 0.0029850006103515625, + "KL/std": 0.5733331441879272, + "epoch": 0.048167539267015703, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09003856778144836, + "fcm_dpo/q_t": 0.4997749328613281, + "grad_norm": 28.17792320251465, + "learning_rate": 2.2916666666666663e-07, + "logits/chosen": -0.6223236918449402, + "logits/rejected": -0.6566183567047119, + "logps/chosen": -282.0877380371094, + "logps/ref_chosen": -282.1807556152344, + "logps/ref_rejected": -265.0758056640625, + "logps/rejected": -265.07281494140625, + "loss": 5.5416, + "margin_dpo/margin_mean": 0.09003835916519165, + "margin_dpo/margin_std": 0.8055158853530884, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.18692398071289062, + "KL/mean": 0.0784425437450409, + "KL/rejected_KL_mean": -0.0300445556640625, + "KL/std": 0.5381811857223511, + "epoch": 0.050261780104712044, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.21695753931999207, + "fcm_dpo/q_t": 0.49945762753486633, + "grad_norm": 29.881175994873047, + "learning_rate": 2.3958333333333335e-07, + "logits/chosen": -0.6491536498069763, + "logits/rejected": -0.5612991452217102, + "logps/chosen": -300.99267578125, + "logps/ref_chosen": -301.17962646484375, + "logps/ref_rejected": -302.12786865234375, + "logps/rejected": -302.15789794921875, + "loss": 5.5366, + "margin_dpo/margin_mean": 0.21695764362812042, + "margin_dpo/margin_std": 0.681999921798706, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.1545238494873047, + "KL/mean": 0.11132223159074783, + "KL/rejected_KL_mean": 0.06811237335205078, + "KL/std": 0.5142117738723755, + "epoch": 0.05235602094240838, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08640654385089874, + "fcm_dpo/q_t": 0.499783992767334, + "grad_norm": 27.276826858520508, + "learning_rate": 2.5e-07, + "logits/chosen": -0.5908087491989136, + "logits/rejected": -0.5986031293869019, + "logps/chosen": -246.59197998046875, + "logps/ref_chosen": -246.74649047851562, + "logps/ref_rejected": -235.55638122558594, + "logps/rejected": -235.48825073242188, + "loss": 5.5418, + "margin_dpo/margin_mean": 0.08640624582767487, + "margin_dpo/margin_std": 0.6666019558906555, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.22557830810546875, + "KL/mean": 0.16001597046852112, + "KL/rejected_KL_mean": 0.09445762634277344, + "KL/std": 0.6255167126655579, + "epoch": 0.05445026178010471, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.13112373650074005, + "fcm_dpo/q_t": 0.49967217445373535, + "grad_norm": 28.602731704711914, + "learning_rate": 2.604166666666667e-07, + "logits/chosen": -0.646224319934845, + "logits/rejected": -0.6599107384681702, + "logps/chosen": -281.97003173828125, + "logps/ref_chosen": -282.1955871582031, + "logps/ref_rejected": -235.3135528564453, + "logps/rejected": -235.21910095214844, + "loss": 5.54, + "margin_dpo/margin_mean": 0.13112422823905945, + "margin_dpo/margin_std": 0.8185635805130005, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.3226490020751953, + "KL/mean": 0.2239154875278473, + "KL/rejected_KL_mean": 0.1251850128173828, + "KL/std": 0.6283989548683167, + "epoch": 0.05654450261780105, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19746924936771393, + "fcm_dpo/q_t": 0.4995063543319702, + "grad_norm": 27.78577423095703, + "learning_rate": 2.708333333333333e-07, + "logits/chosen": -0.6513367891311646, + "logits/rejected": -0.6705080270767212, + "logps/chosen": -323.53369140625, + "logps/ref_chosen": -323.8563537597656, + "logps/ref_rejected": -245.968017578125, + "logps/rejected": -245.84283447265625, + "loss": 5.5374, + "margin_dpo/margin_mean": 0.19746987521648407, + "margin_dpo/margin_std": 0.8737579584121704, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.2768516540527344, + "KL/mean": 0.19153092801570892, + "KL/rejected_KL_mean": 0.10620689392089844, + "KL/std": 0.5605363845825195, + "epoch": 0.05863874345549738, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1706477254629135, + "fcm_dpo/q_t": 0.49957337975502014, + "grad_norm": 26.49478530883789, + "learning_rate": 2.8125e-07, + "logits/chosen": -0.6174635887145996, + "logits/rejected": -0.6264654397964478, + "logps/chosen": -247.9698944091797, + "logps/ref_chosen": -248.24673461914062, + "logps/ref_rejected": -240.0382080078125, + "logps/rejected": -239.9320068359375, + "loss": 5.5384, + "margin_dpo/margin_mean": 0.17064793407917023, + "margin_dpo/margin_std": 0.7738831043243408, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.29106712341308594, + "KL/mean": 0.19878274202346802, + "KL/rejected_KL_mean": 0.10648536682128906, + "KL/std": 0.5895746350288391, + "epoch": 0.060732984293193716, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.18458740413188934, + "fcm_dpo/q_t": 0.4995385706424713, + "grad_norm": 29.785554885864258, + "learning_rate": 2.916666666666667e-07, + "logits/chosen": -0.5842040777206421, + "logits/rejected": -0.6051267385482788, + "logps/chosen": -317.9653625488281, + "logps/ref_chosen": -318.2564392089844, + "logps/ref_rejected": -286.75848388671875, + "logps/rejected": -286.65203857421875, + "loss": 5.5379, + "margin_dpo/margin_mean": 0.18458794057369232, + "margin_dpo/margin_std": 0.7843542098999023, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.3651390075683594, + "KL/mean": 0.22121500968933105, + "KL/rejected_KL_mean": 0.07729148864746094, + "KL/std": 0.6174904704093933, + "epoch": 0.06282722513089005, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.28784090280532837, + "fcm_dpo/q_t": 0.4992803931236267, + "grad_norm": 28.858478546142578, + "learning_rate": 3.020833333333333e-07, + "logits/chosen": -0.6003649830818176, + "logits/rejected": -0.6176969408988953, + "logps/chosen": -252.68399047851562, + "logps/ref_chosen": -253.0491485595703, + "logps/ref_rejected": -261.30029296875, + "logps/rejected": -261.2229919433594, + "loss": 5.5337, + "margin_dpo/margin_mean": 0.2878407835960388, + "margin_dpo/margin_std": 0.8439157605171204, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.41502952575683594, + "KL/mean": 0.2872907519340515, + "KL/rejected_KL_mean": 0.1595611572265625, + "KL/std": 0.7194770574569702, + "epoch": 0.06492146596858639, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.25546789169311523, + "fcm_dpo/q_t": 0.4993613660335541, + "grad_norm": 25.050519943237305, + "learning_rate": 3.1249999999999997e-07, + "logits/chosen": -0.6552233099937439, + "logits/rejected": -0.6901096105575562, + "logps/chosen": -247.73800659179688, + "logps/ref_chosen": -248.15301513671875, + "logps/ref_rejected": -203.17703247070312, + "logps/rejected": -203.01748657226562, + "loss": 5.5351, + "margin_dpo/margin_mean": 0.25546786189079285, + "margin_dpo/margin_std": 0.900585949420929, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.45583343505859375, + "KL/mean": 0.34075260162353516, + "KL/rejected_KL_mean": 0.22568130493164062, + "KL/std": 0.723097026348114, + "epoch": 0.06701570680628273, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2301485389471054, + "fcm_dpo/q_t": 0.49942466616630554, + "grad_norm": 29.70162582397461, + "learning_rate": 3.2291666666666666e-07, + "logits/chosen": -0.6078366637229919, + "logits/rejected": -0.6122292876243591, + "logps/chosen": -305.0841369628906, + "logps/ref_chosen": -305.5399475097656, + "logps/ref_rejected": -267.6527099609375, + "logps/rejected": -267.4270324707031, + "loss": 5.5361, + "margin_dpo/margin_mean": 0.23014913499355316, + "margin_dpo/margin_std": 0.9483384490013123, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.5750503540039062, + "KL/mean": 0.37175193428993225, + "KL/rejected_KL_mean": 0.16846084594726562, + "KL/std": 0.9059445858001709, + "epoch": 0.06910994764397906, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4065783619880676, + "fcm_dpo/q_t": 0.49898362159729004, + "grad_norm": 28.4556941986084, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.640916109085083, + "logits/rejected": -0.6534519195556641, + "logps/chosen": -285.6584777832031, + "logps/ref_chosen": -286.2335205078125, + "logps/ref_rejected": -255.38748168945312, + "logps/rejected": -255.2190399169922, + "loss": 5.5291, + "margin_dpo/margin_mean": 0.4065782427787781, + "margin_dpo/margin_std": 1.132638931274414, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.7782306671142578, + "KL/mean": 0.35225048661231995, + "KL/rejected_KL_mean": -0.07373237609863281, + "KL/std": 1.0198003053665161, + "epoch": 0.0712041884816754, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8519618511199951, + "fcm_dpo/q_t": 0.49787020683288574, + "grad_norm": 31.113811492919922, + "learning_rate": 3.4375e-07, + "logits/chosen": -0.6230222582817078, + "logits/rejected": -0.6323559284210205, + "logps/chosen": -340.8137512207031, + "logps/ref_chosen": -341.5920104980469, + "logps/ref_rejected": -278.8866882324219, + "logps/rejected": -278.9604187011719, + "loss": 5.5113, + "margin_dpo/margin_mean": 0.8519620299339294, + "margin_dpo/margin_std": 1.140161156654358, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.6100578308105469, + "KL/mean": 0.45359697937965393, + "KL/rejected_KL_mean": 0.2971305847167969, + "KL/std": 0.9627859592437744, + "epoch": 0.07329842931937172, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.31292349100112915, + "fcm_dpo/q_t": 0.4992177188396454, + "grad_norm": 26.534446716308594, + "learning_rate": 3.541666666666667e-07, + "logits/chosen": -0.6332972049713135, + "logits/rejected": -0.6537303924560547, + "logps/chosen": -264.469482421875, + "logps/ref_chosen": -265.0795593261719, + "logps/ref_rejected": -264.4876708984375, + "logps/rejected": -264.1905517578125, + "loss": 5.5328, + "margin_dpo/margin_mean": 0.3129233419895172, + "margin_dpo/margin_std": 1.3062032461166382, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.8854637145996094, + "KL/mean": 0.5023590326309204, + "KL/rejected_KL_mean": 0.11924552917480469, + "KL/std": 1.133253812789917, + "epoch": 0.07539267015706806, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7662180662155151, + "fcm_dpo/q_t": 0.4980846047401428, + "grad_norm": 31.921070098876953, + "learning_rate": 3.645833333333333e-07, + "logits/chosen": -0.6024245023727417, + "logits/rejected": -0.6200557947158813, + "logps/chosen": -296.440673828125, + "logps/ref_chosen": -297.3261413574219, + "logps/ref_rejected": -282.09515380859375, + "logps/rejected": -281.97589111328125, + "loss": 5.5148, + "margin_dpo/margin_mean": 0.7662180066108704, + "margin_dpo/margin_std": 1.4552934169769287, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.7702865600585938, + "KL/mean": 0.4680500328540802, + "KL/rejected_KL_mean": 0.1658344268798828, + "KL/std": 1.1320182085037231, + "epoch": 0.0774869109947644, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6044507026672363, + "fcm_dpo/q_t": 0.4984889626502991, + "grad_norm": 31.38983917236328, + "learning_rate": 3.75e-07, + "logits/chosen": -0.6020391583442688, + "logits/rejected": -0.6165008544921875, + "logps/chosen": -313.26373291015625, + "logps/ref_chosen": -314.0340270996094, + "logps/ref_rejected": -299.3437805175781, + "logps/rejected": -299.1779479980469, + "loss": 5.5213, + "margin_dpo/margin_mean": 0.6044489741325378, + "margin_dpo/margin_std": 1.5211684703826904, + "step": 37 + }, + { + "KL/chosen_KL_mean": 1.0551738739013672, + "KL/mean": 0.7106471061706543, + "KL/rejected_KL_mean": 0.36612892150878906, + "KL/std": 1.2290730476379395, + "epoch": 0.07958115183246073, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.689054012298584, + "fcm_dpo/q_t": 0.4982774555683136, + "grad_norm": 28.488887786865234, + "learning_rate": 3.8541666666666665e-07, + "logits/chosen": -0.6347458362579346, + "logits/rejected": -0.6467496156692505, + "logps/chosen": -281.48602294921875, + "logps/ref_chosen": -282.54119873046875, + "logps/ref_rejected": -269.7773132324219, + "logps/rejected": -269.4111633300781, + "loss": 5.5179, + "margin_dpo/margin_mean": 0.6890544891357422, + "margin_dpo/margin_std": 1.4637418985366821, + "step": 38 + }, + { + "KL/chosen_KL_mean": 1.2901058197021484, + "KL/mean": 0.679212212562561, + "KL/rejected_KL_mean": 0.06832122802734375, + "KL/std": 1.531196117401123, + "epoch": 0.08167539267015707, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2217795848846436, + "fcm_dpo/q_t": 0.49694594740867615, + "grad_norm": 29.57316017150879, + "learning_rate": 3.958333333333333e-07, + "logits/chosen": -0.6168429851531982, + "logits/rejected": -0.6308571100234985, + "logps/chosen": -275.48284912109375, + "logps/ref_chosen": -276.7729187011719, + "logps/ref_rejected": -249.95889282226562, + "logps/rejected": -249.89056396484375, + "loss": 5.4968, + "margin_dpo/margin_mean": 1.221779465675354, + "margin_dpo/margin_std": 1.7274365425109863, + "step": 39 + }, + { + "KL/chosen_KL_mean": 1.0404300689697266, + "KL/mean": 0.7055494785308838, + "KL/rejected_KL_mean": 0.3706798553466797, + "KL/std": 1.3352773189544678, + "epoch": 0.08376963350785341, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6697390675544739, + "fcm_dpo/q_t": 0.4983258545398712, + "grad_norm": 27.396766662597656, + "learning_rate": 4.0625e-07, + "logits/chosen": -0.6229257583618164, + "logits/rejected": -0.6586186289787292, + "logps/chosen": -283.26666259765625, + "logps/ref_chosen": -284.30706787109375, + "logps/ref_rejected": -244.4459991455078, + "logps/rejected": -244.07530212402344, + "loss": 5.5188, + "margin_dpo/margin_mean": 0.6697392463684082, + "margin_dpo/margin_std": 1.7444779872894287, + "step": 40 + }, + { + "KL/chosen_KL_mean": 1.0908069610595703, + "KL/mean": 0.7018202543258667, + "KL/rejected_KL_mean": 0.31284141540527344, + "KL/std": 1.4677762985229492, + "epoch": 0.08586387434554973, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7779622077941895, + "fcm_dpo/q_t": 0.49805524945259094, + "grad_norm": 30.210500717163086, + "learning_rate": 4.1666666666666667e-07, + "logits/chosen": -0.6286399364471436, + "logits/rejected": -0.6542515754699707, + "logps/chosen": -292.724365234375, + "logps/ref_chosen": -293.8151550292969, + "logps/ref_rejected": -252.16815185546875, + "logps/rejected": -251.85531616210938, + "loss": 5.5144, + "margin_dpo/margin_mean": 0.7779629826545715, + "margin_dpo/margin_std": 1.7769787311553955, + "step": 41 + }, + { + "KL/chosen_KL_mean": 1.2728042602539062, + "KL/mean": 0.7940388917922974, + "KL/rejected_KL_mean": 0.31528282165527344, + "KL/std": 1.7132642269134521, + "epoch": 0.08795811518324607, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9575194716453552, + "fcm_dpo/q_t": 0.4976065456867218, + "grad_norm": 27.403196334838867, + "learning_rate": 4.270833333333333e-07, + "logits/chosen": -0.6356105804443359, + "logits/rejected": -0.6535137891769409, + "logps/chosen": -251.4874725341797, + "logps/ref_chosen": -252.76023864746094, + "logps/ref_rejected": -261.0414733886719, + "logps/rejected": -260.7261962890625, + "loss": 5.5075, + "margin_dpo/margin_mean": 0.95751953125, + "margin_dpo/margin_std": 2.236936092376709, + "step": 42 + }, + { + "KL/chosen_KL_mean": 1.2807998657226562, + "KL/mean": 0.6640318632125854, + "KL/rejected_KL_mean": 0.04726409912109375, + "KL/std": 1.9099464416503906, + "epoch": 0.09005235602094241, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.233534812927246, + "fcm_dpo/q_t": 0.4969165027141571, + "grad_norm": 29.962818145751953, + "learning_rate": 4.375e-07, + "logits/chosen": -0.6014646887779236, + "logits/rejected": -0.6178789138793945, + "logps/chosen": -315.553955078125, + "logps/ref_chosen": -316.8347473144531, + "logps/ref_rejected": -273.7649230957031, + "logps/rejected": -273.7176513671875, + "loss": 5.4966, + "margin_dpo/margin_mean": 1.2335340976715088, + "margin_dpo/margin_std": 2.2836263179779053, + "step": 43 + }, + { + "KL/chosen_KL_mean": 1.5793323516845703, + "KL/mean": 0.7775524854660034, + "KL/rejected_KL_mean": -0.02422332763671875, + "KL/std": 2.20059871673584, + "epoch": 0.09214659685863874, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6035535335540771, + "fcm_dpo/q_t": 0.49599185585975647, + "grad_norm": 30.827856063842773, + "learning_rate": 4.479166666666667e-07, + "logits/chosen": -0.5976197719573975, + "logits/rejected": -0.5963581800460815, + "logps/chosen": -285.2963562011719, + "logps/ref_chosen": -286.8757019042969, + "logps/ref_rejected": -282.4681396484375, + "logps/rejected": -282.49237060546875, + "loss": 5.4822, + "margin_dpo/margin_mean": 1.6035535335540771, + "margin_dpo/margin_std": 2.972196340560913, + "step": 44 + }, + { + "KL/chosen_KL_mean": 1.7299308776855469, + "KL/mean": 1.0418126583099365, + "KL/rejected_KL_mean": 0.35369300842285156, + "KL/std": 2.250051975250244, + "epoch": 0.09424083769633508, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3762341737747192, + "fcm_dpo/q_t": 0.4965605139732361, + "grad_norm": 28.878467559814453, + "learning_rate": 4.5833333333333327e-07, + "logits/chosen": -0.688926100730896, + "logits/rejected": -0.7126062512397766, + "logps/chosen": -322.533447265625, + "logps/ref_chosen": -324.2633972167969, + "logps/ref_rejected": -293.09466552734375, + "logps/rejected": -292.740966796875, + "loss": 5.4912, + "margin_dpo/margin_mean": 1.376233696937561, + "margin_dpo/margin_std": 2.813117742538452, + "step": 45 + }, + { + "KL/chosen_KL_mean": 1.6946258544921875, + "KL/mean": 0.9494956135749817, + "KL/rejected_KL_mean": 0.20435523986816406, + "KL/std": 2.433075189590454, + "epoch": 0.09633507853403141, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4902722835540771, + "fcm_dpo/q_t": 0.4962751865386963, + "grad_norm": 30.186275482177734, + "learning_rate": 4.6874999999999996e-07, + "logits/chosen": -0.6262366771697998, + "logits/rejected": -0.6397067904472351, + "logps/chosen": -296.64111328125, + "logps/ref_chosen": -298.3357238769531, + "logps/ref_rejected": -267.66204833984375, + "logps/rejected": -267.4576721191406, + "loss": 5.4865, + "margin_dpo/margin_mean": 1.4902722835540771, + "margin_dpo/margin_std": 2.6397438049316406, + "step": 46 + }, + { + "KL/chosen_KL_mean": 1.5152873992919922, + "KL/mean": 1.0190459489822388, + "KL/rejected_KL_mean": 0.5227985382080078, + "KL/std": 2.292161464691162, + "epoch": 0.09842931937172775, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9924949407577515, + "fcm_dpo/q_t": 0.4975191056728363, + "grad_norm": 26.3062801361084, + "learning_rate": 4.791666666666667e-07, + "logits/chosen": -0.6064034700393677, + "logits/rejected": -0.6271387934684753, + "logps/chosen": -261.0516357421875, + "logps/ref_chosen": -262.5669250488281, + "logps/ref_rejected": -258.70989990234375, + "logps/rejected": -258.18707275390625, + "loss": 5.5068, + "margin_dpo/margin_mean": 0.9924947619438171, + "margin_dpo/margin_std": 3.2504637241363525, + "step": 47 + }, + { + "KL/chosen_KL_mean": 1.820608139038086, + "KL/mean": 1.0710346698760986, + "KL/rejected_KL_mean": 0.32146644592285156, + "KL/std": 2.579556465148926, + "epoch": 0.10052356020942409, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4991388320922852, + "fcm_dpo/q_t": 0.4962541460990906, + "grad_norm": 27.537498474121094, + "learning_rate": 4.895833333333333e-07, + "logits/chosen": -0.6123904585838318, + "logits/rejected": -0.637353777885437, + "logps/chosen": -267.67266845703125, + "logps/ref_chosen": -269.4932556152344, + "logps/ref_rejected": -241.888916015625, + "logps/rejected": -241.56744384765625, + "loss": 5.4865, + "margin_dpo/margin_mean": 1.4991381168365479, + "margin_dpo/margin_std": 2.9511523246765137, + "step": 48 + }, + { + "KL/chosen_KL_mean": 2.2462005615234375, + "KL/mean": 1.3681857585906982, + "KL/rejected_KL_mean": 0.49018096923828125, + "KL/std": 2.621547222137451, + "epoch": 0.10261780104712041, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7560153007507324, + "fcm_dpo/q_t": 0.4956113398075104, + "grad_norm": 27.621536254882812, + "learning_rate": 5e-07, + "logits/chosen": -0.664618194103241, + "logits/rejected": -0.6490676999092102, + "logps/chosen": -255.63827514648438, + "logps/ref_chosen": -257.8844909667969, + "logps/ref_rejected": -256.8912048339844, + "logps/rejected": -256.4010009765625, + "loss": 5.4765, + "margin_dpo/margin_mean": 1.7560160160064697, + "margin_dpo/margin_std": 3.3718807697296143, + "step": 49 + }, + { + "KL/chosen_KL_mean": 1.9965705871582031, + "KL/mean": 1.436202883720398, + "KL/rejected_KL_mean": 0.8758354187011719, + "KL/std": 2.9903759956359863, + "epoch": 0.10471204188481675, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1207308769226074, + "fcm_dpo/q_t": 0.4971994161605835, + "grad_norm": 27.850839614868164, + "learning_rate": 4.999932966293553e-07, + "logits/chosen": -0.6214494109153748, + "logits/rejected": -0.642817497253418, + "logps/chosen": -299.63226318359375, + "logps/ref_chosen": -301.62884521484375, + "logps/ref_rejected": -298.2716064453125, + "logps/rejected": -297.3957824707031, + "loss": 5.5021, + "margin_dpo/margin_mean": 1.1207301616668701, + "margin_dpo/margin_std": 3.682468891143799, + "step": 50 + }, + { + "KL/chosen_KL_mean": 2.1178417205810547, + "KL/mean": 1.1750433444976807, + "KL/rejected_KL_mean": 0.23224449157714844, + "KL/std": 3.547469139099121, + "epoch": 0.1068062827225131, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8856074810028076, + "fcm_dpo/q_t": 0.49528858065605164, + "grad_norm": 29.11636734008789, + "learning_rate": 4.999731868769026e-07, + "logits/chosen": -0.6039535403251648, + "logits/rejected": -0.5983304381370544, + "logps/chosen": -267.2545166015625, + "logps/ref_chosen": -269.37237548828125, + "logps/ref_rejected": -297.0167541503906, + "logps/rejected": -296.7845153808594, + "loss": 5.4725, + "margin_dpo/margin_mean": 1.8856079578399658, + "margin_dpo/margin_std": 4.448060512542725, + "step": 51 + }, + { + "KL/chosen_KL_mean": 2.7982959747314453, + "KL/mean": 1.2421739101409912, + "KL/rejected_KL_mean": -0.3139476776123047, + "KL/std": 3.763486862182617, + "epoch": 0.10890052356020942, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.1122491359710693, + "fcm_dpo/q_t": 0.49222618341445923, + "grad_norm": 30.2871150970459, + "learning_rate": 4.99939671821067e-07, + "logits/chosen": -0.6552134156227112, + "logits/rejected": -0.6618373990058899, + "logps/chosen": -304.1044921875, + "logps/ref_chosen": -306.9028015136719, + "logps/ref_rejected": -281.24737548828125, + "logps/rejected": -281.561279296875, + "loss": 5.4241, + "margin_dpo/margin_mean": 3.1122488975524902, + "margin_dpo/margin_std": 4.741540431976318, + "step": 52 + }, + { + "KL/chosen_KL_mean": 2.7981128692626953, + "KL/mean": 1.7105417251586914, + "KL/rejected_KL_mean": 0.6229763031005859, + "KL/std": 4.075300216674805, + "epoch": 0.11099476439790576, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.1751344203948975, + "fcm_dpo/q_t": 0.49456626176834106, + "grad_norm": 30.046728134155273, + "learning_rate": 4.998927532591591e-07, + "logits/chosen": -0.6532727479934692, + "logits/rejected": -0.6938633322715759, + "logps/chosen": -283.1778564453125, + "logps/ref_chosen": -285.9759521484375, + "logps/ref_rejected": -273.9073486328125, + "logps/rejected": -273.28436279296875, + "loss": 5.4616, + "margin_dpo/margin_mean": 2.17513370513916, + "margin_dpo/margin_std": 5.007012367248535, + "step": 53 + }, + { + "KL/chosen_KL_mean": 2.679128646850586, + "KL/mean": 1.732741355895996, + "KL/rejected_KL_mean": 0.7863483428955078, + "KL/std": 4.326586723327637, + "epoch": 0.1130890052356021, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8927712440490723, + "fcm_dpo/q_t": 0.4952733814716339, + "grad_norm": 26.467914581298828, + "learning_rate": 4.998324337072792e-07, + "logits/chosen": -0.690001368522644, + "logits/rejected": -0.6982678174972534, + "logps/chosen": -303.8254699707031, + "logps/ref_chosen": -306.504638671875, + "logps/ref_rejected": -272.67431640625, + "logps/rejected": -271.8879699707031, + "loss": 5.473, + "margin_dpo/margin_mean": 1.89277184009552, + "margin_dpo/margin_std": 5.428363800048828, + "step": 54 + }, + { + "KL/chosen_KL_mean": 1.8250560760498047, + "KL/mean": 0.6974931955337524, + "KL/rejected_KL_mean": -0.43007469177246094, + "KL/std": 3.710113525390625, + "epoch": 0.11518324607329843, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.255131244659424, + "fcm_dpo/q_t": 0.49436742067337036, + "grad_norm": 24.798433303833008, + "learning_rate": 4.997587164001815e-07, + "logits/chosen": -0.6320837736129761, + "logits/rejected": -0.635603129863739, + "logps/chosen": -220.50509643554688, + "logps/ref_chosen": -222.33013916015625, + "logps/ref_rejected": -206.59571838378906, + "logps/rejected": -207.0258026123047, + "loss": 5.458, + "margin_dpo/margin_mean": 2.255131959915161, + "margin_dpo/margin_std": 4.947845458984375, + "step": 55 + }, + { + "KL/chosen_KL_mean": 2.8548545837402344, + "KL/mean": 1.2973169088363647, + "KL/rejected_KL_mean": -0.26021766662597656, + "KL/std": 4.783367156982422, + "epoch": 0.11727748691099477, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.1150808334350586, + "fcm_dpo/q_t": 0.49222424626350403, + "grad_norm": 27.468914031982422, + "learning_rate": 4.996716052911017e-07, + "logits/chosen": -0.613375723361969, + "logits/rejected": -0.6283761262893677, + "logps/chosen": -247.62330627441406, + "logps/ref_chosen": -250.47816467285156, + "logps/ref_rejected": -228.25848388671875, + "logps/rejected": -228.51870727539062, + "loss": 5.4259, + "margin_dpo/margin_mean": 3.115081310272217, + "margin_dpo/margin_std": 5.996671676635742, + "step": 56 + }, + { + "KL/chosen_KL_mean": 3.899179458618164, + "KL/mean": 1.7846465110778809, + "KL/rejected_KL_mean": -0.3298816680908203, + "KL/std": 4.845379829406738, + "epoch": 0.1193717277486911, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.229069709777832, + "fcm_dpo/q_t": 0.4894401431083679, + "grad_norm": 30.87196159362793, + "learning_rate": 4.99571105051544e-07, + "logits/chosen": -0.7000694274902344, + "logits/rejected": -0.6709764003753662, + "logps/chosen": -311.2203369140625, + "logps/ref_chosen": -315.1195373535156, + "logps/ref_rejected": -272.755615234375, + "logps/rejected": -273.0854797363281, + "loss": 5.3813, + "margin_dpo/margin_mean": 4.229069709777832, + "margin_dpo/margin_std": 5.674056529998779, + "step": 57 + }, + { + "KL/chosen_KL_mean": 2.4763240814208984, + "KL/mean": 1.3078148365020752, + "KL/rejected_KL_mean": 0.13930511474609375, + "KL/std": 4.4075093269348145, + "epoch": 0.12146596858638743, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.337027072906494, + "fcm_dpo/q_t": 0.494160920381546, + "grad_norm": 27.59241485595703, + "learning_rate": 4.994572210710314e-07, + "logits/chosen": -0.6156286597251892, + "logits/rejected": -0.6398040652275085, + "logps/chosen": -262.705322265625, + "logps/ref_chosen": -265.1816711425781, + "logps/ref_rejected": -268.2203369140625, + "logps/rejected": -268.0810241699219, + "loss": 5.4557, + "margin_dpo/margin_mean": 2.3370275497436523, + "margin_dpo/margin_std": 5.594066619873047, + "step": 58 + }, + { + "KL/chosen_KL_mean": 2.1144371032714844, + "KL/mean": 1.332619071006775, + "KL/rejected_KL_mean": 0.5508079528808594, + "KL/std": 5.286384582519531, + "epoch": 0.12356020942408377, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5636342763900757, + "fcm_dpo/q_t": 0.4960969388484955, + "grad_norm": 30.200071334838867, + "learning_rate": 4.993299594568162e-07, + "logits/chosen": -0.6096245646476746, + "logits/rejected": -0.603198766708374, + "logps/chosen": -284.2395324707031, + "logps/ref_chosen": -286.35394287109375, + "logps/ref_rejected": -260.6757507324219, + "logps/rejected": -260.12493896484375, + "loss": 5.4887, + "margin_dpo/margin_mean": 1.5636343955993652, + "margin_dpo/margin_std": 7.07401180267334, + "step": 59 + }, + { + "KL/chosen_KL_mean": 2.8281993865966797, + "KL/mean": 1.1985671520233154, + "KL/rejected_KL_mean": -0.43106842041015625, + "KL/std": 5.1828532218933105, + "epoch": 0.1256544502617801, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.2592642307281494, + "fcm_dpo/q_t": 0.49186116456985474, + "grad_norm": 28.115793228149414, + "learning_rate": 4.991893270335525e-07, + "logits/chosen": -0.6829241514205933, + "logits/rejected": -0.7077679634094238, + "logps/chosen": -255.92039489746094, + "logps/ref_chosen": -258.74859619140625, + "logps/ref_rejected": -255.04893493652344, + "logps/rejected": -255.47998046875, + "loss": 5.4214, + "margin_dpo/margin_mean": 3.2592647075653076, + "margin_dpo/margin_std": 7.173513412475586, + "step": 60 + }, + { + "KL/chosen_KL_mean": 3.054656982421875, + "KL/mean": 1.4622215032577515, + "KL/rejected_KL_mean": -0.13021087646484375, + "KL/std": 5.500098705291748, + "epoch": 0.12774869109947645, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.184875965118408, + "fcm_dpo/q_t": 0.492051899433136, + "grad_norm": 29.790935516357422, + "learning_rate": 4.990353313429303e-07, + "logits/chosen": -0.6545593738555908, + "logits/rejected": -0.6720077395439148, + "logps/chosen": -275.41326904296875, + "logps/ref_chosen": -278.4678955078125, + "logps/ref_rejected": -252.02720642089844, + "logps/rejected": -252.1573944091797, + "loss": 5.4254, + "margin_dpo/margin_mean": 3.184875965118408, + "margin_dpo/margin_std": 7.548828125, + "step": 61 + }, + { + "KL/chosen_KL_mean": 3.975238800048828, + "KL/mean": 2.29765248298645, + "KL/rejected_KL_mean": 0.6200656890869141, + "KL/std": 5.711119174957275, + "epoch": 0.12984293193717278, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.3551716804504395, + "fcm_dpo/q_t": 0.49163132905960083, + "grad_norm": 26.7069091796875, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": -0.6063894629478455, + "logits/rejected": -0.6477152705192566, + "logps/chosen": -268.9490661621094, + "logps/ref_chosen": -272.92431640625, + "logps/ref_rejected": -260.7935485839844, + "logps/rejected": -260.1734924316406, + "loss": 5.4178, + "margin_dpo/margin_mean": 3.3551721572875977, + "margin_dpo/margin_std": 7.388532638549805, + "step": 62 + }, + { + "KL/chosen_KL_mean": 3.386953353881836, + "KL/mean": 1.6372132301330566, + "KL/rejected_KL_mean": -0.11253166198730469, + "KL/std": 5.6143670082092285, + "epoch": 0.1319371727748691, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.4994750022888184, + "fcm_dpo/q_t": 0.4912656843662262, + "grad_norm": 28.21631622314453, + "learning_rate": 4.986872839090852e-07, + "logits/chosen": -0.6588255167007446, + "logits/rejected": -0.6650886535644531, + "logps/chosen": -273.7020568847656, + "logps/ref_chosen": -277.0889892578125, + "logps/ref_rejected": -273.3413391113281, + "logps/rejected": -273.453857421875, + "loss": 5.4121, + "margin_dpo/margin_mean": 3.4994747638702393, + "margin_dpo/margin_std": 7.25181770324707, + "step": 63 + }, + { + "KL/chosen_KL_mean": 4.168355941772461, + "KL/mean": 2.0174996852874756, + "KL/rejected_KL_mean": -0.13335037231445312, + "KL/std": 6.28251314163208, + "epoch": 0.13403141361256546, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.301701545715332, + "fcm_dpo/q_t": 0.48926711082458496, + "grad_norm": 28.383853912353516, + "learning_rate": 4.9849325083059e-07, + "logits/chosen": -0.6357841491699219, + "logits/rejected": -0.6312809586524963, + "logps/chosen": -279.6561279296875, + "logps/ref_chosen": -283.8244934082031, + "logps/ref_rejected": -263.29351806640625, + "logps/rejected": -263.4268798828125, + "loss": 5.3824, + "margin_dpo/margin_mean": 4.301700592041016, + "margin_dpo/margin_std": 8.25213623046875, + "step": 64 + }, + { + "KL/chosen_KL_mean": 3.4636306762695312, + "KL/mean": 1.8112151622772217, + "KL/rejected_KL_mean": 0.15880584716796875, + "KL/std": 5.756163597106934, + "epoch": 0.13612565445026178, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.3048229217529297, + "fcm_dpo/q_t": 0.4917513132095337, + "grad_norm": 27.825502395629883, + "learning_rate": 4.982858918131906e-07, + "logits/chosen": -0.7022321820259094, + "logits/rejected": -0.6702457070350647, + "logps/chosen": -261.40631103515625, + "logps/ref_chosen": -264.8699645996094, + "logps/ref_rejected": -268.5076904296875, + "logps/rejected": -268.348876953125, + "loss": 5.4199, + "margin_dpo/margin_mean": 3.3048219680786133, + "margin_dpo/margin_std": 7.397376537322998, + "step": 65 + }, + { + "KL/chosen_KL_mean": 3.1544952392578125, + "KL/mean": 1.0874898433685303, + "KL/rejected_KL_mean": -0.9795093536376953, + "KL/std": 7.427680015563965, + "epoch": 0.1382198952879581, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.133997440338135, + "fcm_dpo/q_t": 0.4896990954875946, + "grad_norm": 27.89104461669922, + "learning_rate": 4.980652179769217e-07, + "logits/chosen": -0.6843776702880859, + "logits/rejected": -0.704654335975647, + "logps/chosen": -269.7738342285156, + "logps/ref_chosen": -272.9283142089844, + "logps/ref_rejected": -280.94696044921875, + "logps/rejected": -281.9264831542969, + "loss": 5.3922, + "margin_dpo/margin_mean": 4.133997917175293, + "margin_dpo/margin_std": 10.16850471496582, + "step": 66 + }, + { + "KL/chosen_KL_mean": 3.9878292083740234, + "KL/mean": 2.169799566268921, + "KL/rejected_KL_mean": 0.3517570495605469, + "KL/std": 7.131447792053223, + "epoch": 0.14031413612565444, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 3.6360793113708496, + "fcm_dpo/q_t": 0.4909287989139557, + "grad_norm": 25.531761169433594, + "learning_rate": 4.978312411558517e-07, + "logits/chosen": -0.6986119747161865, + "logits/rejected": -0.7322227954864502, + "logps/chosen": -262.1990966796875, + "logps/ref_chosen": -266.18695068359375, + "logps/ref_rejected": -250.17405700683594, + "logps/rejected": -249.82232666015625, + "loss": 5.4094, + "margin_dpo/margin_mean": 3.6360793113708496, + "margin_dpo/margin_std": 8.844635963439941, + "step": 67 + }, + { + "KL/chosen_KL_mean": 3.0627079010009766, + "KL/mean": 0.719912588596344, + "KL/rejected_KL_mean": -1.6228828430175781, + "KL/std": 8.057466506958008, + "epoch": 0.1424083769633508, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.685594081878662, + "fcm_dpo/q_t": 0.488344669342041, + "grad_norm": 27.940174102783203, + "learning_rate": 4.975839738974473e-07, + "logits/chosen": -0.6814758777618408, + "logits/rejected": -0.6943728923797607, + "logps/chosen": -294.8758850097656, + "logps/ref_chosen": -297.9385986328125, + "logps/ref_rejected": -261.5141296386719, + "logps/rejected": -263.1369934082031, + "loss": 5.3719, + "margin_dpo/margin_mean": 4.685595512390137, + "margin_dpo/margin_std": 10.544692039489746, + "step": 68 + }, + { + "KL/chosen_KL_mean": 3.918996810913086, + "KL/mean": 0.8691984415054321, + "KL/rejected_KL_mean": -2.1805973052978516, + "KL/std": 7.882665634155273, + "epoch": 0.14450261780104712, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.09959602355957, + "fcm_dpo/q_t": 0.48479950428009033, + "grad_norm": 28.539579391479492, + "learning_rate": 4.97323429461901e-07, + "logits/chosen": -0.6883444786071777, + "logits/rejected": -0.719520092010498, + "logps/chosen": -261.69854736328125, + "logps/ref_chosen": -265.6175231933594, + "logps/ref_rejected": -236.8287353515625, + "logps/rejected": -239.00933837890625, + "loss": 5.3149, + "margin_dpo/margin_mean": 6.09959602355957, + "margin_dpo/margin_std": 9.689201354980469, + "step": 69 + }, + { + "KL/chosen_KL_mean": 4.362461090087891, + "KL/mean": 1.4138339757919312, + "KL/rejected_KL_mean": -1.5347957611083984, + "KL/std": 8.69215202331543, + "epoch": 0.14659685863874344, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.897259712219238, + "fcm_dpo/q_t": 0.48531487584114075, + "grad_norm": 28.618436813354492, + "learning_rate": 4.970496218214204e-07, + "logits/chosen": -0.6700032353401184, + "logits/rejected": -0.7025509476661682, + "logps/chosen": -291.86346435546875, + "logps/ref_chosen": -296.2259216308594, + "logps/ref_rejected": -254.68496704101562, + "logps/rejected": -256.2197570800781, + "loss": 5.3258, + "margin_dpo/margin_mean": 5.897259712219238, + "margin_dpo/margin_std": 10.952218055725098, + "step": 70 + }, + { + "KL/chosen_KL_mean": 5.089008331298828, + "KL/mean": 2.5183145999908447, + "KL/rejected_KL_mean": -0.05237388610839844, + "KL/std": 9.370198249816895, + "epoch": 0.1486910994764398, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.141389846801758, + "fcm_dpo/q_t": 0.4871872365474701, + "grad_norm": 28.453304290771484, + "learning_rate": 4.967625656594781e-07, + "logits/chosen": -0.6482348442077637, + "logits/rejected": -0.6368006467819214, + "logps/chosen": -283.8382263183594, + "logps/ref_chosen": -288.92724609375, + "logps/ref_rejected": -278.6405334472656, + "logps/rejected": -278.69287109375, + "loss": 5.3587, + "margin_dpo/margin_mean": 5.141388893127441, + "margin_dpo/margin_std": 12.29645824432373, + "step": 71 + }, + { + "KL/chosen_KL_mean": 3.3922672271728516, + "KL/mean": 0.7981147766113281, + "KL/rejected_KL_mean": -1.7960338592529297, + "KL/std": 8.441047668457031, + "epoch": 0.15078534031413612, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.188295364379883, + "fcm_dpo/q_t": 0.48706212639808655, + "grad_norm": 28.272216796875, + "learning_rate": 4.964622763700252e-07, + "logits/chosen": -0.6958415508270264, + "logits/rejected": -0.7072215676307678, + "logps/chosen": -233.65304565429688, + "logps/ref_chosen": -237.0452880859375, + "logps/ref_rejected": -252.7946319580078, + "logps/rejected": -254.59066772460938, + "loss": 5.3528, + "margin_dpo/margin_mean": 5.188295364379883, + "margin_dpo/margin_std": 10.39709186553955, + "step": 72 + }, + { + "KL/chosen_KL_mean": 4.309686660766602, + "KL/mean": 1.9369821548461914, + "KL/rejected_KL_mean": -0.43572235107421875, + "KL/std": 8.945561408996582, + "epoch": 0.15287958115183245, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.74540901184082, + "fcm_dpo/q_t": 0.4881640076637268, + "grad_norm": 27.904314041137695, + "learning_rate": 4.961487700566646e-07, + "logits/chosen": -0.6588871479034424, + "logits/rejected": -0.6765344142913818, + "logps/chosen": -268.743408203125, + "logps/ref_chosen": -273.0531005859375, + "logps/ref_rejected": -246.8330841064453, + "logps/rejected": -247.26881408691406, + "loss": 5.3739, + "margin_dpo/margin_mean": 4.745408058166504, + "margin_dpo/margin_std": 12.261763572692871, + "step": 73 + }, + { + "KL/chosen_KL_mean": 3.3888206481933594, + "KL/mean": 1.1340489387512207, + "KL/rejected_KL_mean": -1.1207256317138672, + "KL/std": 9.382542610168457, + "epoch": 0.1549738219895288, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 4.5095534324646, + "fcm_dpo/q_t": 0.48877227306365967, + "grad_norm": 30.305757522583008, + "learning_rate": 4.958220635317885e-07, + "logits/chosen": -0.7255840301513672, + "logits/rejected": -0.7040022611618042, + "logps/chosen": -338.8930969238281, + "logps/ref_chosen": -342.2818908691406, + "logps/ref_rejected": -330.0293884277344, + "logps/rejected": -331.15008544921875, + "loss": 5.3817, + "margin_dpo/margin_mean": 4.509552478790283, + "margin_dpo/margin_std": 11.70348834991455, + "step": 74 + }, + { + "KL/chosen_KL_mean": 4.573873519897461, + "KL/mean": 1.2532798051834106, + "KL/rejected_KL_mean": -2.0673141479492188, + "KL/std": 9.401840209960938, + "epoch": 0.15706806282722513, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.64119291305542, + "fcm_dpo/q_t": 0.4834534525871277, + "grad_norm": 29.507638931274414, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": -0.6492323279380798, + "logits/rejected": -0.6508209109306335, + "logps/chosen": -262.290283203125, + "logps/ref_chosen": -266.8641662597656, + "logps/ref_rejected": -276.8699951171875, + "logps/rejected": -278.93731689453125, + "loss": 5.2973, + "margin_dpo/margin_mean": 6.641193389892578, + "margin_dpo/margin_std": 10.963482856750488, + "step": 75 + }, + { + "KL/chosen_KL_mean": 4.076141357421875, + "KL/mean": 0.49301546812057495, + "KL/rejected_KL_mean": -3.0901031494140625, + "KL/std": 9.934422492980957, + "epoch": 0.15916230366492146, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.166243553161621, + "fcm_dpo/q_t": 0.48219114542007446, + "grad_norm": 29.160371780395508, + "learning_rate": 4.951291206355559e-07, + "logits/chosen": -0.7237194180488586, + "logits/rejected": -0.731726884841919, + "logps/chosen": -277.09844970703125, + "logps/ref_chosen": -281.174560546875, + "logps/ref_rejected": -263.6067199707031, + "logps/rejected": -266.6968078613281, + "loss": 5.2812, + "margin_dpo/margin_mean": 7.166243553161621, + "margin_dpo/margin_std": 12.741656303405762, + "step": 76 + }, + { + "KL/chosen_KL_mean": 3.6962413787841797, + "KL/mean": 0.7907732725143433, + "KL/rejected_KL_mean": -2.1146984100341797, + "KL/std": 11.032858848571777, + "epoch": 0.1612565445026178, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.81093692779541, + "fcm_dpo/q_t": 0.48557212948799133, + "grad_norm": 33.117801666259766, + "learning_rate": 4.947629214246236e-07, + "logits/chosen": -0.5612236857414246, + "logits/rejected": -0.5700373649597168, + "logps/chosen": -302.3990478515625, + "logps/ref_chosen": -306.09527587890625, + "logps/ref_rejected": -253.49569702148438, + "logps/rejected": -255.61038208007812, + "loss": 5.3373, + "margin_dpo/margin_mean": 5.810935974121094, + "margin_dpo/margin_std": 14.009510040283203, + "step": 77 + }, + { + "KL/chosen_KL_mean": 3.9560623168945312, + "KL/mean": -0.42408621311187744, + "KL/rejected_KL_mean": -4.804229736328125, + "KL/std": 11.058279037475586, + "epoch": 0.16335078534031414, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.760295867919922, + "fcm_dpo/q_t": 0.4782665967941284, + "grad_norm": 29.5765323638916, + "learning_rate": 4.943835963210323e-07, + "logits/chosen": -0.6872852444648743, + "logits/rejected": -0.6824613213539124, + "logps/chosen": -252.9462890625, + "logps/ref_chosen": -256.90234375, + "logps/ref_rejected": -211.57154846191406, + "logps/rejected": -216.37579345703125, + "loss": 5.2243, + "margin_dpo/margin_mean": 8.760295867919922, + "margin_dpo/margin_std": 14.509359359741211, + "step": 78 + }, + { + "KL/chosen_KL_mean": 3.094745635986328, + "KL/mean": -1.0207563638687134, + "KL/rejected_KL_mean": -5.136257171630859, + "KL/std": 11.751511573791504, + "epoch": 0.16544502617801046, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.231000900268555, + "fcm_dpo/q_t": 0.47957462072372437, + "grad_norm": 29.715789794921875, + "learning_rate": 4.939911656668361e-07, + "logits/chosen": -0.6479459404945374, + "logits/rejected": -0.6681416630744934, + "logps/chosen": -263.1788330078125, + "logps/ref_chosen": -266.2735595703125, + "logps/ref_rejected": -251.57257080078125, + "logps/rejected": -256.7088317871094, + "loss": 5.2467, + "margin_dpo/margin_mean": 8.231000900268555, + "margin_dpo/margin_std": 14.964641571044922, + "step": 79 + }, + { + "KL/chosen_KL_mean": 1.8658809661865234, + "KL/mean": -1.5003409385681152, + "KL/rejected_KL_mean": -4.866567611694336, + "KL/std": 11.38090991973877, + "epoch": 0.16753926701570682, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 6.732439994812012, + "fcm_dpo/q_t": 0.48329317569732666, + "grad_norm": 29.368417739868164, + "learning_rate": 4.935856505068998e-07, + "logits/chosen": -0.6606127619743347, + "logits/rejected": -0.6897295713424683, + "logps/chosen": -285.9851379394531, + "logps/ref_chosen": -287.8509826660156, + "logps/ref_rejected": -256.0766296386719, + "logps/rejected": -260.94317626953125, + "loss": 5.3003, + "margin_dpo/margin_mean": 6.732440948486328, + "margin_dpo/margin_std": 13.253829002380371, + "step": 80 + }, + { + "KL/chosen_KL_mean": 2.3929805755615234, + "KL/mean": -1.201179027557373, + "KL/rejected_KL_mean": -4.795331954956055, + "KL/std": 12.221700668334961, + "epoch": 0.16963350785340314, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.188300132751465, + "fcm_dpo/q_t": 0.4821750223636627, + "grad_norm": 28.137069702148438, + "learning_rate": 4.93167072587771e-07, + "logits/chosen": -0.639922022819519, + "logits/rejected": -0.6389482021331787, + "logps/chosen": -266.1302490234375, + "logps/ref_chosen": -268.5232238769531, + "logps/ref_rejected": -237.81137084960938, + "logps/rejected": -242.606689453125, + "loss": 5.2926, + "margin_dpo/margin_mean": 7.188301086425781, + "margin_dpo/margin_std": 16.771045684814453, + "step": 81 + }, + { + "KL/chosen_KL_mean": 2.4245166778564453, + "KL/mean": -1.560113549232483, + "KL/rejected_KL_mean": -5.544746398925781, + "KL/std": 12.169999122619629, + "epoch": 0.17172774869109947, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.9692702293396, + "fcm_dpo/q_t": 0.48028764128685, + "grad_norm": 27.79278564453125, + "learning_rate": 4.92735454356513e-07, + "logits/chosen": -0.7311412692070007, + "logits/rejected": -0.7389302849769592, + "logps/chosen": -276.939453125, + "logps/ref_chosen": -279.36395263671875, + "logps/ref_rejected": -236.51365661621094, + "logps/rejected": -242.0584259033203, + "loss": 5.2612, + "margin_dpo/margin_mean": 7.969269275665283, + "margin_dpo/margin_std": 15.728371620178223, + "step": 82 + }, + { + "KL/chosen_KL_mean": 0.2986888885498047, + "KL/mean": -4.056522846221924, + "KL/rejected_KL_mean": -8.411733627319336, + "KL/std": 13.574930191040039, + "epoch": 0.17382198952879582, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.710416793823242, + "fcm_dpo/q_t": 0.4783723056316376, + "grad_norm": 30.74686050415039, + "learning_rate": 4.922908189595017e-07, + "logits/chosen": -0.6830888390541077, + "logits/rejected": -0.6660097241401672, + "logps/chosen": -273.9205322265625, + "logps/ref_chosen": -274.21923828125, + "logps/ref_rejected": -276.2212219238281, + "logps/rejected": -284.6329345703125, + "loss": 5.2404, + "margin_dpo/margin_mean": 8.71041488647461, + "margin_dpo/margin_std": 18.00052261352539, + "step": 83 + }, + { + "KL/chosen_KL_mean": 0.45940589904785156, + "KL/mean": -3.1783106327056885, + "KL/rejected_KL_mean": -6.816030502319336, + "KL/std": 13.303352355957031, + "epoch": 0.17591623036649215, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 7.275437355041504, + "fcm_dpo/q_t": 0.4819525480270386, + "grad_norm": 29.886608123779297, + "learning_rate": 4.918331902411841e-07, + "logits/chosen": -0.7300011515617371, + "logits/rejected": -0.7440951466560364, + "logps/chosen": -293.9381408691406, + "logps/ref_chosen": -294.3975524902344, + "logps/ref_rejected": -279.81884765625, + "logps/rejected": -286.6348571777344, + "loss": 5.2907, + "margin_dpo/margin_mean": 7.275436878204346, + "margin_dpo/margin_std": 16.904508590698242, + "step": 84 + }, + { + "KL/chosen_KL_mean": -1.547414779663086, + "KL/mean": -4.410724639892578, + "KL/rejected_KL_mean": -7.274040222167969, + "KL/std": 12.268994331359863, + "epoch": 0.17801047120418848, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 5.726627349853516, + "fcm_dpo/q_t": 0.4857904314994812, + "grad_norm": 29.356456756591797, + "learning_rate": 4.913625927427995e-07, + "logits/chosen": -0.6729651093482971, + "logits/rejected": -0.681312620639801, + "logps/chosen": -245.20960998535156, + "logps/ref_chosen": -243.66220092773438, + "logps/ref_rejected": -263.9421691894531, + "logps/rejected": -271.2162170410156, + "loss": 5.3462, + "margin_dpo/margin_mean": 5.726626873016357, + "margin_dpo/margin_std": 15.388749122619629, + "step": 85 + }, + { + "KL/chosen_KL_mean": 1.1150398254394531, + "KL/mean": -3.287755012512207, + "KL/rejected_KL_mean": -7.690544128417969, + "KL/std": 13.255000114440918, + "epoch": 0.18010471204188483, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.805594444274902, + "fcm_dpo/q_t": 0.47813674807548523, + "grad_norm": 35.038631439208984, + "learning_rate": 4.908790517010636e-07, + "logits/chosen": -0.6957335472106934, + "logits/rejected": -0.689771294593811, + "logps/chosen": -308.315673828125, + "logps/ref_chosen": -309.4306945800781, + "logps/ref_rejected": -290.91278076171875, + "logps/rejected": -298.60333251953125, + "loss": 5.2323, + "margin_dpo/margin_mean": 8.805594444274902, + "margin_dpo/margin_std": 17.30478286743164, + "step": 86 + }, + { + "KL/chosen_KL_mean": -0.07946395874023438, + "KL/mean": -5.250776290893555, + "KL/rejected_KL_mean": -10.42209243774414, + "KL/std": 15.329278945922852, + "epoch": 0.18219895287958116, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.342628479003906, + "fcm_dpo/q_t": 0.47446227073669434, + "grad_norm": 29.616622924804688, + "learning_rate": 4.903825930468148e-07, + "logits/chosen": -0.7527617812156677, + "logits/rejected": -0.7467857599258423, + "logps/chosen": -278.107177734375, + "logps/ref_chosen": -278.0277099609375, + "logps/ref_rejected": -245.70123291015625, + "logps/rejected": -256.1233215332031, + "loss": 5.1807, + "margin_dpo/margin_mean": 10.34262752532959, + "margin_dpo/margin_std": 18.891633987426758, + "step": 87 + }, + { + "KL/chosen_KL_mean": -2.065469741821289, + "KL/mean": -6.552918910980225, + "KL/rejected_KL_mean": -11.040365219116211, + "KL/std": 15.065473556518555, + "epoch": 0.18429319371727748, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.974905014038086, + "fcm_dpo/q_t": 0.47781187295913696, + "grad_norm": 28.785079956054688, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": -0.7596137523651123, + "logits/rejected": -0.7751888632774353, + "logps/chosen": -268.58026123046875, + "logps/ref_chosen": -266.5148010253906, + "logps/ref_rejected": -265.90081787109375, + "logps/rejected": -276.9411926269531, + "loss": 5.2321, + "margin_dpo/margin_mean": 8.974905014038086, + "margin_dpo/margin_std": 19.049829483032227, + "step": 88 + }, + { + "KL/chosen_KL_mean": 0.024187088012695312, + "KL/mean": -4.604069709777832, + "KL/rejected_KL_mean": -9.232324600219727, + "KL/std": 14.210840225219727, + "epoch": 0.18638743455497384, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.256507873535156, + "fcm_dpo/q_t": 0.47706693410873413, + "grad_norm": 30.544334411621094, + "learning_rate": 4.893510300863676e-07, + "logits/chosen": -0.7496181726455688, + "logits/rejected": -0.7401238083839417, + "logps/chosen": -265.66510009765625, + "logps/ref_chosen": -265.6893005371094, + "logps/ref_rejected": -251.49314880371094, + "logps/rejected": -260.7254638671875, + "loss": 5.2206, + "margin_dpo/margin_mean": 9.256505966186523, + "margin_dpo/margin_std": 18.1968994140625, + "step": 89 + }, + { + "KL/chosen_KL_mean": -1.1707649230957031, + "KL/mean": -5.307417869567871, + "KL/rejected_KL_mean": -9.444063186645508, + "KL/std": 14.324629783630371, + "epoch": 0.18848167539267016, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 8.273297309875488, + "fcm_dpo/q_t": 0.47948795557022095, + "grad_norm": 29.870534896850586, + "learning_rate": 4.8881598109976e-07, + "logits/chosen": -0.7418705224990845, + "logits/rejected": -0.7506077885627747, + "logps/chosen": -308.59576416015625, + "logps/ref_chosen": -307.4250183105469, + "logps/ref_rejected": -265.7172546386719, + "logps/rejected": -275.16131591796875, + "loss": 5.255, + "margin_dpo/margin_mean": 8.273297309875488, + "margin_dpo/margin_std": 17.62283706665039, + "step": 90 + }, + { + "KL/chosen_KL_mean": -2.0556297302246094, + "KL/mean": -6.845937252044678, + "KL/rejected_KL_mean": -11.636245727539062, + "KL/std": 14.768171310424805, + "epoch": 0.1905759162303665, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.580626487731934, + "fcm_dpo/q_t": 0.4763721525669098, + "grad_norm": 32.8757438659668, + "learning_rate": 4.882681251368548e-07, + "logits/chosen": -0.6754969358444214, + "logits/rejected": -0.6923132538795471, + "logps/chosen": -237.79660034179688, + "logps/ref_chosen": -235.74098205566406, + "logps/ref_rejected": -226.6428985595703, + "logps/rejected": -238.27914428710938, + "loss": 5.213, + "margin_dpo/margin_mean": 9.580625534057617, + "margin_dpo/margin_std": 19.14481544494629, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.073078155517578, + "KL/mean": -8.132434844970703, + "KL/rejected_KL_mean": -13.191781997680664, + "KL/std": 17.528228759765625, + "epoch": 0.19267015706806281, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.118701934814453, + "fcm_dpo/q_t": 0.47499901056289673, + "grad_norm": 33.32277297973633, + "learning_rate": 4.877074915775048e-07, + "logits/chosen": -0.7399584054946899, + "logits/rejected": -0.7235689759254456, + "logps/chosen": -286.5206298828125, + "logps/ref_chosen": -283.4475402832031, + "logps/ref_rejected": -273.134033203125, + "logps/rejected": -286.3258056640625, + "loss": 5.1989, + "margin_dpo/margin_mean": 10.118700981140137, + "margin_dpo/margin_std": 21.361736297607422, + "step": 92 + }, + { + "KL/chosen_KL_mean": -2.37042236328125, + "KL/mean": -7.15866756439209, + "KL/rejected_KL_mean": -11.946908950805664, + "KL/std": 15.452308654785156, + "epoch": 0.19476439790575917, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.576484680175781, + "fcm_dpo/q_t": 0.47637003660202026, + "grad_norm": 28.583599090576172, + "learning_rate": 4.871341104867864e-07, + "logits/chosen": -0.7276648879051208, + "logits/rejected": -0.7507765293121338, + "logps/chosen": -235.70758056640625, + "logps/ref_chosen": -233.33714294433594, + "logps/ref_rejected": -230.54273986816406, + "logps/rejected": -242.48963928222656, + "loss": 5.2119, + "margin_dpo/margin_mean": 9.576486587524414, + "margin_dpo/margin_std": 19.396862030029297, + "step": 93 + }, + { + "KL/chosen_KL_mean": -2.370729446411133, + "KL/mean": -7.069452285766602, + "KL/rejected_KL_mean": -11.768178939819336, + "KL/std": 16.41796875, + "epoch": 0.1968586387434555, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.397444725036621, + "fcm_dpo/q_t": 0.47674643993377686, + "grad_norm": 31.590362548828125, + "learning_rate": 4.865480126133871e-07, + "logits/chosen": -0.6804681420326233, + "logits/rejected": -0.701062798500061, + "logps/chosen": -297.0235290527344, + "logps/ref_chosen": -294.6528015136719, + "logps/ref_rejected": -283.657958984375, + "logps/rejected": -295.4261779785156, + "loss": 5.229, + "margin_dpo/margin_mean": 9.397444725036621, + "margin_dpo/margin_std": 21.770397186279297, + "step": 94 + }, + { + "KL/chosen_KL_mean": -3.2703723907470703, + "KL/mean": -8.370147705078125, + "KL/rejected_KL_mean": -13.469928741455078, + "KL/std": 18.059005737304688, + "epoch": 0.19895287958115182, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.199569702148438, + "fcm_dpo/q_t": 0.47488659620285034, + "grad_norm": 33.58381652832031, + "learning_rate": 4.859492293879573e-07, + "logits/chosen": -0.7297713756561279, + "logits/rejected": -0.7513625621795654, + "logps/chosen": -314.9400634765625, + "logps/ref_chosen": -311.6697082519531, + "logps/ref_rejected": -262.7471923828125, + "logps/rejected": -276.2171325683594, + "loss": 5.2047, + "margin_dpo/margin_mean": 10.199568748474121, + "margin_dpo/margin_std": 22.428693771362305, + "step": 95 + }, + { + "KL/chosen_KL_mean": -4.741254806518555, + "KL/mean": -9.848160743713379, + "KL/rejected_KL_mean": -14.955074310302734, + "KL/std": 19.324003219604492, + "epoch": 0.20104712041884817, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 10.213811874389648, + "fcm_dpo/q_t": 0.47487515211105347, + "grad_norm": 32.13194274902344, + "learning_rate": 4.853377929214243e-07, + "logits/chosen": -0.7099624276161194, + "logits/rejected": -0.7225924134254456, + "logps/chosen": -287.2972412109375, + "logps/ref_chosen": -282.55596923828125, + "logps/ref_rejected": -242.71588134765625, + "logps/rejected": -257.67095947265625, + "loss": 5.2055, + "margin_dpo/margin_mean": 10.213811874389648, + "margin_dpo/margin_std": 23.361492156982422, + "step": 96 + }, + { + "KL/chosen_KL_mean": -4.432626724243164, + "KL/mean": -10.405980110168457, + "KL/rejected_KL_mean": -16.379331588745117, + "KL/std": 17.279687881469727, + "epoch": 0.2031413612565445, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 11.946697235107422, + "fcm_dpo/q_t": 0.4705726206302643, + "grad_norm": 32.038726806640625, + "learning_rate": 4.847137360032699e-07, + "logits/chosen": -0.7478024959564209, + "logits/rejected": -0.7351346015930176, + "logps/chosen": -308.0104064941406, + "logps/ref_chosen": -303.57781982421875, + "logps/ref_rejected": -264.22491455078125, + "logps/rejected": -280.6042785644531, + "loss": 5.134, + "margin_dpo/margin_mean": 11.946695327758789, + "margin_dpo/margin_std": 22.253314971923828, + "step": 97 + }, + { + "KL/chosen_KL_mean": -4.91217041015625, + "KL/mean": -11.056485176086426, + "KL/rejected_KL_mean": -17.200815200805664, + "KL/std": 18.921445846557617, + "epoch": 0.20523560209424083, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.288653373718262, + "fcm_dpo/q_t": 0.46991679072380066, + "grad_norm": 34.590091705322266, + "learning_rate": 4.84077092099773e-07, + "logits/chosen": -0.773546576499939, + "logits/rejected": -0.7858595848083496, + "logps/chosen": -291.7425231933594, + "logps/ref_chosen": -286.8303527832031, + "logps/ref_rejected": -278.08331298828125, + "logps/rejected": -295.28411865234375, + "loss": 5.1334, + "margin_dpo/margin_mean": 12.288653373718262, + "margin_dpo/margin_std": 22.73413848876953, + "step": 98 + }, + { + "KL/chosen_KL_mean": -5.357412338256836, + "KL/mean": -11.63817310333252, + "KL/rejected_KL_mean": -17.918926239013672, + "KL/std": 19.809310913085938, + "epoch": 0.20732984293193718, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.561515808105469, + "fcm_dpo/q_t": 0.46920517086982727, + "grad_norm": 32.64725112915039, + "learning_rate": 4.834278953522137e-07, + "logits/chosen": -0.7336536645889282, + "logits/rejected": -0.7463732957839966, + "logps/chosen": -285.27862548828125, + "logps/ref_chosen": -279.92120361328125, + "logps/ref_rejected": -250.3365478515625, + "logps/rejected": -268.2554626464844, + "loss": 5.1374, + "margin_dpo/margin_mean": 12.561513900756836, + "margin_dpo/margin_std": 27.08693504333496, + "step": 99 + }, + { + "KL/chosen_KL_mean": -7.981771469116211, + "KL/mean": -14.22314167022705, + "KL/rejected_KL_mean": -20.464502334594727, + "KL/std": 22.023868560791016, + "epoch": 0.2094240837696335, + "fcm_dpo/beta": 0.009999998845160007, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.482728958129883, + "fcm_dpo/q_t": 0.46943894028663635, + "grad_norm": 35.665283203125, + "learning_rate": 4.827661805750437e-07, + "logits/chosen": -0.7793423533439636, + "logits/rejected": -0.792355477809906, + "logps/chosen": -304.8094482421875, + "logps/ref_chosen": -296.8276672363281, + "logps/ref_rejected": -275.56146240234375, + "logps/rejected": -296.0259704589844, + "loss": 5.1259, + "margin_dpo/margin_mean": 12.482728958129883, + "margin_dpo/margin_std": 24.164777755737305, + "step": 100 + }, + { + "KL/chosen_KL_mean": -5.265691757202148, + "KL/mean": -12.787820816040039, + "KL/rejected_KL_mean": -20.309947967529297, + "KL/std": 22.83624267578125, + "epoch": 0.21151832460732983, + "fcm_dpo/beta": 0.010065114125609398, + "fcm_dpo/delta": 0.012955582700669765, + "fcm_dpo/margin": 15.044249534606934, + "fcm_dpo/q_t": 0.46306145191192627, + "grad_norm": 32.623504638671875, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": -0.7783185839653015, + "logits/rejected": -0.7923865914344788, + "logps/chosen": -258.0077209472656, + "logps/ref_chosen": -252.74203491210938, + "logps/ref_rejected": -276.4185485839844, + "logps/rejected": -296.728515625, + "loss": 5.0388, + "margin_dpo/margin_mean": 15.044251441955566, + "margin_dpo/margin_std": 26.482027053833008, + "step": 101 + }, + { + "KL/chosen_KL_mean": -5.2807159423828125, + "KL/mean": -12.680944442749023, + "KL/rejected_KL_mean": -20.0811710357666, + "KL/std": 19.747940063476562, + "epoch": 0.2136125654450262, + "fcm_dpo/beta": 0.010104183107614517, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.800459861755371, + "fcm_dpo/q_t": 0.4631401300430298, + "grad_norm": 32.70088577270508, + "learning_rate": 4.814053395442932e-07, + "logits/chosen": -0.748501718044281, + "logits/rejected": -0.7446244359016418, + "logps/chosen": -224.8344268798828, + "logps/ref_chosen": -219.5537109375, + "logps/ref_rejected": -231.90853881835938, + "logps/rejected": -251.98971557617188, + "loss": 5.0391, + "margin_dpo/margin_mean": 14.800461769104004, + "margin_dpo/margin_std": 24.98674201965332, + "step": 102 + }, + { + "KL/chosen_KL_mean": -4.796598434448242, + "KL/mean": -11.604303359985352, + "KL/rejected_KL_mean": -18.41200828552246, + "KL/std": 21.641616821289062, + "epoch": 0.2157068062827225, + "fcm_dpo/beta": 0.010104183107614517, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.615408897399902, + "fcm_dpo/q_t": 0.46653062105178833, + "grad_norm": 33.16029357910156, + "learning_rate": 4.807062862684873e-07, + "logits/chosen": -0.7775754332542419, + "logits/rejected": -0.7745975852012634, + "logps/chosen": -264.4716491699219, + "logps/ref_chosen": -259.6750793457031, + "logps/ref_rejected": -278.7400817871094, + "logps/rejected": -297.1520690917969, + "loss": 5.087, + "margin_dpo/margin_mean": 13.615410804748535, + "margin_dpo/margin_std": 25.769948959350586, + "step": 103 + }, + { + "KL/chosen_KL_mean": -9.229019165039062, + "KL/mean": -14.208805084228516, + "KL/rejected_KL_mean": -19.188581466674805, + "KL/std": 22.482053756713867, + "epoch": 0.21780104712041884, + "fcm_dpo/beta": 0.010104183107614517, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 9.959566116333008, + "fcm_dpo/q_t": 0.4753492772579193, + "grad_norm": 33.56724166870117, + "learning_rate": 4.799948609147061e-07, + "logits/chosen": -0.7666143178939819, + "logits/rejected": -0.7724973559379578, + "logps/chosen": -277.20318603515625, + "logps/ref_chosen": -267.9741516113281, + "logps/ref_rejected": -230.5306396484375, + "logps/rejected": -249.7191925048828, + "loss": 5.2263, + "margin_dpo/margin_mean": 9.959567070007324, + "margin_dpo/margin_std": 26.30889129638672, + "step": 104 + }, + { + "KL/chosen_KL_mean": -5.367591857910156, + "KL/mean": -15.618803977966309, + "KL/rejected_KL_mean": -25.870025634765625, + "KL/std": 25.24723243713379, + "epoch": 0.2198952879581152, + "fcm_dpo/beta": 0.010265516117215157, + "fcm_dpo/delta": 0.03177732229232788, + "fcm_dpo/margin": 20.502422332763672, + "fcm_dpo/q_t": 0.44872522354125977, + "grad_norm": 35.49409484863281, + "learning_rate": 4.792711016345321e-07, + "logits/chosen": -0.7507005929946899, + "logits/rejected": -0.7593094706535339, + "logps/chosen": -327.6224365234375, + "logps/ref_chosen": -322.25482177734375, + "logps/ref_rejected": -279.02978515625, + "logps/rejected": -304.8998107910156, + "loss": 4.8339, + "margin_dpo/margin_mean": 20.502424240112305, + "margin_dpo/margin_std": 27.04251480102539, + "step": 105 + }, + { + "KL/chosen_KL_mean": -12.565797805786133, + "KL/mean": -18.833768844604492, + "KL/rejected_KL_mean": -25.101734161376953, + "KL/std": 23.593149185180664, + "epoch": 0.22198952879581152, + "fcm_dpo/beta": 0.010364342480897903, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 12.535932540893555, + "fcm_dpo/q_t": 0.46829015016555786, + "grad_norm": 39.71345901489258, + "learning_rate": 4.785350472409791e-07, + "logits/chosen": -0.7475220561027527, + "logits/rejected": -0.7846206426620483, + "logps/chosen": -308.7236022949219, + "logps/ref_chosen": -296.15777587890625, + "logps/ref_rejected": -266.2691650390625, + "logps/rejected": -291.3708801269531, + "loss": 5.141, + "margin_dpo/margin_mean": 12.535932540893555, + "margin_dpo/margin_std": 29.484294891357422, + "step": 106 + }, + { + "KL/chosen_KL_mean": -4.345104217529297, + "KL/mean": -14.212882995605469, + "KL/rejected_KL_mean": -24.08066749572754, + "KL/std": 22.098743438720703, + "epoch": 0.22408376963350785, + "fcm_dpo/beta": 0.010364342480897903, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.735557556152344, + "fcm_dpo/q_t": 0.45006412267684937, + "grad_norm": 39.31448745727539, + "learning_rate": 4.777867372064105e-07, + "logits/chosen": -0.7761357426643372, + "logits/rejected": -0.7689798474311829, + "logps/chosen": -311.3414001464844, + "logps/ref_chosen": -306.996337890625, + "logps/ref_rejected": -296.79412841796875, + "logps/rejected": -320.8747863769531, + "loss": 4.8556, + "margin_dpo/margin_mean": 19.735557556152344, + "margin_dpo/margin_std": 27.56524658203125, + "step": 107 + }, + { + "KL/chosen_KL_mean": -5.493007659912109, + "KL/mean": -14.60330867767334, + "KL/rejected_KL_mean": -23.713605880737305, + "KL/std": 25.141176223754883, + "epoch": 0.2261780104712042, + "fcm_dpo/beta": 0.010364342480897903, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.220611572265625, + "fcm_dpo/q_t": 0.45436638593673706, + "grad_norm": 105.63594055175781, + "learning_rate": 4.770262116604223e-07, + "logits/chosen": -0.7658447027206421, + "logits/rejected": -0.7774515151977539, + "logps/chosen": -300.6456604003906, + "logps/ref_chosen": -295.1526794433594, + "logps/ref_rejected": -235.974853515625, + "logps/rejected": -259.6884460449219, + "loss": 4.9372, + "margin_dpo/margin_mean": 18.220611572265625, + "margin_dpo/margin_std": 30.389331817626953, + "step": 108 + }, + { + "KL/chosen_KL_mean": -8.395013809204102, + "KL/mean": -18.529293060302734, + "KL/rejected_KL_mean": -28.663572311401367, + "KL/std": 25.285425186157227, + "epoch": 0.22827225130890053, + "fcm_dpo/beta": 0.010364342480897903, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 20.268556594848633, + "fcm_dpo/q_t": 0.4494061768054962, + "grad_norm": 39.464012145996094, + "learning_rate": 4.7625351138769166e-07, + "logits/chosen": -0.7855874300003052, + "logits/rejected": -0.7813213467597961, + "logps/chosen": -334.31982421875, + "logps/ref_chosen": -325.9248046875, + "logps/ref_rejected": -279.15423583984375, + "logps/rejected": -307.81781005859375, + "loss": 4.8598, + "margin_dpo/margin_mean": 20.2685546875, + "margin_dpo/margin_std": 31.691814422607422, + "step": 109 + }, + { + "KL/chosen_KL_mean": -8.880786895751953, + "KL/mean": -18.487215042114258, + "KL/rejected_KL_mean": -28.093652725219727, + "KL/std": 27.44156837463379, + "epoch": 0.23036649214659685, + "fcm_dpo/beta": 0.010364342480897903, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 19.212865829467773, + "fcm_dpo/q_t": 0.45193198323249817, + "grad_norm": 35.9247932434082, + "learning_rate": 4.75468677825789e-07, + "logits/chosen": -0.8031843900680542, + "logits/rejected": -0.7917722463607788, + "logps/chosen": -283.32000732421875, + "logps/ref_chosen": -274.439208984375, + "logps/ref_rejected": -260.0552062988281, + "logps/rejected": -288.14886474609375, + "loss": 4.9139, + "margin_dpo/margin_mean": 19.212865829467773, + "margin_dpo/margin_std": 33.88175964355469, + "step": 110 + }, + { + "KL/chosen_KL_mean": -9.789043426513672, + "KL/mean": -19.974393844604492, + "KL/rejected_KL_mean": -30.159746170043945, + "KL/std": 28.102462768554688, + "epoch": 0.2324607329842932, + "fcm_dpo/beta": 0.010417740792036057, + "fcm_dpo/delta": 0.017835039645433426, + "fcm_dpo/margin": 20.370704650878906, + "fcm_dpo/q_t": 0.4491615891456604, + "grad_norm": 39.79193878173828, + "learning_rate": 4.7467175306295647e-07, + "logits/chosen": -0.8448382616043091, + "logits/rejected": -0.8246790170669556, + "logps/chosen": -339.02520751953125, + "logps/ref_chosen": -329.2361755371094, + "logps/ref_rejected": -287.82830810546875, + "logps/rejected": -317.988037109375, + "loss": 4.8781, + "margin_dpo/margin_mean": 20.370704650878906, + "margin_dpo/margin_std": 34.60567092895508, + "step": 111 + }, + { + "KL/chosen_KL_mean": -15.530427932739258, + "KL/mean": -22.135120391845703, + "KL/rejected_KL_mean": -28.739818572998047, + "KL/std": 27.021099090576172, + "epoch": 0.23455497382198953, + "fcm_dpo/beta": 0.010554548352956772, + "fcm_dpo/delta": 0.006524841766804457, + "fcm_dpo/margin": 13.209390640258789, + "fcm_dpo/q_t": 0.46666276454925537, + "grad_norm": 41.61723327636719, + "learning_rate": 4.7386277983585053e-07, + "logits/chosen": -0.7402675151824951, + "logits/rejected": -0.7702385783195496, + "logps/chosen": -272.5898132324219, + "logps/ref_chosen": -257.0593566894531, + "logps/ref_rejected": -272.9595031738281, + "logps/rejected": -301.6993103027344, + "loss": 5.1718, + "margin_dpo/margin_mean": 13.209390640258789, + "margin_dpo/margin_std": 36.302940368652344, + "step": 112 + }, + { + "KL/chosen_KL_mean": -12.931312561035156, + "KL/mean": -25.255403518676758, + "KL/rejected_KL_mean": -37.579498291015625, + "KL/std": 31.106075286865234, + "epoch": 0.23664921465968586, + "fcm_dpo/beta": 0.010605855844914913, + "fcm_dpo/delta": 0.008770487271249294, + "fcm_dpo/margin": 24.648176193237305, + "fcm_dpo/q_t": 0.43847501277923584, + "grad_norm": 42.69111251831055, + "learning_rate": 4.7304180152725024e-07, + "logits/chosen": -0.801536500453949, + "logits/rejected": -0.8040984869003296, + "logps/chosen": -298.97296142578125, + "logps/ref_chosen": -286.0416564941406, + "logps/ref_rejected": -270.374267578125, + "logps/rejected": -307.9537658691406, + "loss": 4.7466, + "margin_dpo/margin_mean": 24.648176193237305, + "margin_dpo/margin_std": 39.36397171020508, + "step": 113 + }, + { + "KL/chosen_KL_mean": -17.070255279541016, + "KL/mean": -23.929719924926758, + "KL/rejected_KL_mean": -30.78919219970703, + "KL/std": 28.087270736694336, + "epoch": 0.2387434554973822, + "fcm_dpo/beta": 0.01064271479845047, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 13.718945503234863, + "fcm_dpo/q_t": 0.46457406878471375, + "grad_norm": 42.539310455322266, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": -0.8641890287399292, + "logits/rejected": -0.8660076856613159, + "logps/chosen": -277.0787353515625, + "logps/ref_chosen": -260.0084533691406, + "logps/ref_rejected": -246.67190551757812, + "logps/rejected": -277.4610900878906, + "loss": 5.1358, + "margin_dpo/margin_mean": 13.718944549560547, + "margin_dpo/margin_std": 35.54159927368164, + "step": 114 + }, + { + "KL/chosen_KL_mean": -18.9876766204834, + "KL/mean": -25.9890079498291, + "KL/rejected_KL_mean": -32.99033737182617, + "KL/std": 30.873493194580078, + "epoch": 0.24083769633507854, + "fcm_dpo/beta": 0.010650977492332458, + "fcm_dpo/delta": 0.0012931342935189605, + "fcm_dpo/margin": 14.002666473388672, + "fcm_dpo/q_t": 0.4652654826641083, + "grad_norm": 46.39145278930664, + "learning_rate": 4.7136400641330245e-07, + "logits/chosen": -0.856718122959137, + "logits/rejected": -0.817272424697876, + "logps/chosen": -318.41064453125, + "logps/ref_chosen": -299.4229736328125, + "logps/ref_rejected": -272.1186828613281, + "logps/rejected": -305.1090393066406, + "loss": 5.1502, + "margin_dpo/margin_mean": 14.002666473388672, + "margin_dpo/margin_std": 38.7987174987793, + "step": 115 + }, + { + "KL/chosen_KL_mean": -15.55940055847168, + "KL/mean": -24.596479415893555, + "KL/rejected_KL_mean": -33.633567810058594, + "KL/std": 28.916170120239258, + "epoch": 0.24293193717277486, + "fcm_dpo/beta": 0.010653730481863022, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 18.074167251586914, + "fcm_dpo/q_t": 0.4535205662250519, + "grad_norm": 40.90885925292969, + "learning_rate": 4.70507279583015e-07, + "logits/chosen": -0.8591501116752625, + "logits/rejected": -0.8206615447998047, + "logps/chosen": -294.8233337402344, + "logps/ref_chosen": -279.263916015625, + "logps/ref_rejected": -253.6192169189453, + "logps/rejected": -287.2527770996094, + "loss": 4.9579, + "margin_dpo/margin_mean": 18.074169158935547, + "margin_dpo/margin_std": 35.237911224365234, + "step": 116 + }, + { + "KL/chosen_KL_mean": -18.99713897705078, + "KL/mean": -29.38507652282715, + "KL/rejected_KL_mean": -39.77302169799805, + "KL/std": 27.603759765625, + "epoch": 0.2450261780104712, + "fcm_dpo/beta": 0.010707004927098751, + "fcm_dpo/delta": 0.0019599078223109245, + "fcm_dpo/margin": 20.775876998901367, + "fcm_dpo/q_t": 0.44658583402633667, + "grad_norm": 51.663761138916016, + "learning_rate": 4.6963872761652834e-07, + "logits/chosen": -0.8078407645225525, + "logits/rejected": -0.8070172071456909, + "logps/chosen": -278.2220153808594, + "logps/ref_chosen": -259.2248840332031, + "logps/ref_rejected": -229.3042755126953, + "logps/rejected": -269.0773010253906, + "loss": 4.8498, + "margin_dpo/margin_mean": 20.775875091552734, + "margin_dpo/margin_std": 31.987648010253906, + "step": 117 + }, + { + "KL/chosen_KL_mean": -25.057470321655273, + "KL/mean": -36.63970184326172, + "KL/rejected_KL_mean": -48.22193145751953, + "KL/std": 34.87174987792969, + "epoch": 0.24712041884816754, + "fcm_dpo/beta": 0.010724391788244247, + "fcm_dpo/delta": 0.009311456233263016, + "fcm_dpo/margin": 23.16446304321289, + "fcm_dpo/q_t": 0.44190675020217896, + "grad_norm": 48.20987319946289, + "learning_rate": 4.687583970916486e-07, + "logits/chosen": -0.8008370399475098, + "logits/rejected": -0.7879800200462341, + "logps/chosen": -292.128173828125, + "logps/ref_chosen": -267.0707092285156, + "logps/ref_rejected": -272.7322082519531, + "logps/rejected": -320.9541320800781, + "loss": 4.833, + "margin_dpo/margin_mean": 23.164464950561523, + "margin_dpo/margin_std": 43.23383712768555, + "step": 118 + }, + { + "KL/chosen_KL_mean": -30.545703887939453, + "KL/mean": -39.11827087402344, + "KL/rejected_KL_mean": -47.69084930419922, + "KL/std": 34.09941482543945, + "epoch": 0.24921465968586387, + "fcm_dpo/beta": 0.010751364752650261, + "fcm_dpo/delta": 0.0010554725304245949, + "fcm_dpo/margin": 17.145132064819336, + "fcm_dpo/q_t": 0.45693254470825195, + "grad_norm": 48.39207077026367, + "learning_rate": 4.6786633521783005e-07, + "logits/chosen": -0.8896147012710571, + "logits/rejected": -0.8895549774169922, + "logps/chosen": -355.2223815917969, + "logps/ref_chosen": -324.6766357421875, + "logps/ref_rejected": -306.0322265625, + "logps/rejected": -353.72308349609375, + "loss": 5.0604, + "margin_dpo/margin_mean": 17.145130157470703, + "margin_dpo/margin_std": 41.501827239990234, + "step": 119 + }, + { + "KL/chosen_KL_mean": -27.356718063354492, + "KL/mean": -37.051090240478516, + "KL/rejected_KL_mean": -46.745460510253906, + "KL/std": 33.13863754272461, + "epoch": 0.2513089005235602, + "fcm_dpo/beta": 0.010827319696545601, + "fcm_dpo/delta": 0.009227736853063107, + "fcm_dpo/margin": 19.38873863220215, + "fcm_dpo/q_t": 0.4509553909301758, + "grad_norm": 44.932960510253906, + "learning_rate": 4.669625898336438e-07, + "logits/chosen": -0.8416849970817566, + "logits/rejected": -0.8531475067138672, + "logps/chosen": -342.6184997558594, + "logps/ref_chosen": -315.2617492675781, + "logps/ref_rejected": -265.32501220703125, + "logps/rejected": -312.0704345703125, + "loss": 4.9718, + "margin_dpo/margin_mean": 19.38874053955078, + "margin_dpo/margin_std": 41.262054443359375, + "step": 120 + }, + { + "KL/chosen_KL_mean": -31.884214401245117, + "KL/mean": -38.95367431640625, + "KL/rejected_KL_mean": -46.02313995361328, + "KL/std": 31.924612045288086, + "epoch": 0.2534031413612565, + "fcm_dpo/beta": 0.010839032009243965, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 14.138921737670898, + "fcm_dpo/q_t": 0.464445561170578, + "grad_norm": 60.977882385253906, + "learning_rate": 4.6604720940421207e-07, + "logits/chosen": -0.8611897230148315, + "logits/rejected": -0.875971257686615, + "logps/chosen": -254.8802947998047, + "logps/ref_chosen": -222.99609375, + "logps/ref_rejected": -226.92860412597656, + "logps/rejected": -272.95172119140625, + "loss": 5.1531, + "margin_dpo/margin_mean": 14.138921737670898, + "margin_dpo/margin_std": 40.55924606323242, + "step": 121 + }, + { + "KL/chosen_KL_mean": -32.263099670410156, + "KL/mean": -42.11972427368164, + "KL/rejected_KL_mean": -51.97634506225586, + "KL/std": 37.34040451049805, + "epoch": 0.2554973821989529, + "fcm_dpo/beta": 0.010961100459098816, + "fcm_dpo/delta": 0.01673881895840168, + "fcm_dpo/margin": 19.7132511138916, + "fcm_dpo/q_t": 0.44972339272499084, + "grad_norm": 58.791770935058594, + "learning_rate": 4.651202430186092e-07, + "logits/chosen": -0.9088048934936523, + "logits/rejected": -0.8704884648323059, + "logps/chosen": -308.2893981933594, + "logps/ref_chosen": -276.02630615234375, + "logps/ref_rejected": -277.97418212890625, + "logps/rejected": -329.9505310058594, + "loss": 4.9927, + "margin_dpo/margin_mean": 19.713253021240234, + "margin_dpo/margin_std": 46.560882568359375, + "step": 122 + }, + { + "KL/chosen_KL_mean": -25.356151580810547, + "KL/mean": -38.11783218383789, + "KL/rejected_KL_mean": -50.8795280456543, + "KL/std": 36.9088249206543, + "epoch": 0.25759162303664923, + "fcm_dpo/beta": 0.011069193482398987, + "fcm_dpo/delta": 0.024628084152936935, + "fcm_dpo/margin": 25.523366928100586, + "fcm_dpo/q_t": 0.4335595369338989, + "grad_norm": 52.54579544067383, + "learning_rate": 4.6418174038722924e-07, + "logits/chosen": -0.827944278717041, + "logits/rejected": -0.818522036075592, + "logps/chosen": -353.5107421875, + "logps/ref_chosen": -328.1546325683594, + "logps/ref_rejected": -280.6911315917969, + "logps/rejected": -331.5706481933594, + "loss": 4.7223, + "margin_dpo/margin_mean": 25.523366928100586, + "margin_dpo/margin_std": 42.302734375, + "step": 123 + }, + { + "KL/chosen_KL_mean": -26.102346420288086, + "KL/mean": -37.06385040283203, + "KL/rejected_KL_mean": -48.025367736816406, + "KL/std": 35.34791564941406, + "epoch": 0.25968586387434556, + "fcm_dpo/beta": 0.011221332475543022, + "fcm_dpo/delta": 0.0027072872035205364, + "fcm_dpo/margin": 21.92302131652832, + "fcm_dpo/q_t": 0.44157105684280396, + "grad_norm": 55.5904426574707, + "learning_rate": 4.6323175183912023e-07, + "logits/chosen": -0.8537352085113525, + "logits/rejected": -0.818031907081604, + "logps/chosen": -301.7985534667969, + "logps/ref_chosen": -275.6961975097656, + "logps/ref_rejected": -225.361572265625, + "logps/rejected": -273.386962890625, + "loss": 4.8314, + "margin_dpo/margin_mean": 21.923023223876953, + "margin_dpo/margin_std": 38.72266387939453, + "step": 124 + }, + { + "KL/chosen_KL_mean": -27.758771896362305, + "KL/mean": -37.67707061767578, + "KL/rejected_KL_mean": -47.595367431640625, + "KL/std": 36.744564056396484, + "epoch": 0.2617801047120419, + "fcm_dpo/beta": 0.01130264438688755, + "fcm_dpo/delta": 0.0206233449280262, + "fcm_dpo/margin": 19.836593627929688, + "fcm_dpo/q_t": 0.44899219274520874, + "grad_norm": 58.402217864990234, + "learning_rate": 4.6227032831928483e-07, + "logits/chosen": -0.8152500987052917, + "logits/rejected": -0.7693508863449097, + "logps/chosen": -305.8285217285156, + "logps/ref_chosen": -278.06976318359375, + "logps/ref_rejected": -265.63873291015625, + "logps/rejected": -313.234130859375, + "loss": 5.0234, + "margin_dpo/margin_mean": 19.836593627929688, + "margin_dpo/margin_std": 50.21840286254883, + "step": 125 + }, + { + "KL/chosen_KL_mean": -23.98495864868164, + "KL/mean": -35.167537689208984, + "KL/rejected_KL_mean": -46.350120544433594, + "KL/std": 38.36277770996094, + "epoch": 0.2638743455497382, + "fcm_dpo/beta": 0.011509076692163944, + "fcm_dpo/delta": 0.01648152619600296, + "fcm_dpo/margin": 22.365156173706055, + "fcm_dpo/q_t": 0.44002822041511536, + "grad_norm": 56.53205871582031, + "learning_rate": 4.612975213859487e-07, + "logits/chosen": -0.8276666402816772, + "logits/rejected": -0.8373324275016785, + "logps/chosen": -345.38104248046875, + "logps/ref_chosen": -321.3960876464844, + "logps/ref_rejected": -285.37664794921875, + "logps/rejected": -331.72674560546875, + "loss": 4.8493, + "margin_dpo/margin_mean": 22.365154266357422, + "margin_dpo/margin_std": 44.759033203125, + "step": 126 + }, + { + "KL/chosen_KL_mean": -22.506771087646484, + "KL/mean": -35.692604064941406, + "KL/rejected_KL_mean": -48.87844467163086, + "KL/std": 39.546775817871094, + "epoch": 0.26596858638743454, + "fcm_dpo/beta": 0.011503228917717934, + "fcm_dpo/delta": -0.021350204944610596, + "fcm_dpo/margin": 26.37167739868164, + "fcm_dpo/q_t": 0.43013378977775574, + "grad_norm": 56.40062713623047, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": -0.9043377637863159, + "logits/rejected": -0.8537869453430176, + "logps/chosen": -329.0655212402344, + "logps/ref_chosen": -306.55877685546875, + "logps/ref_rejected": -274.8651428222656, + "logps/rejected": -323.74359130859375, + "loss": 4.7248, + "margin_dpo/margin_mean": 26.371681213378906, + "margin_dpo/margin_std": 45.39989471435547, + "step": 127 + }, + { + "KL/chosen_KL_mean": -17.200231552124023, + "KL/mean": -32.34983825683594, + "KL/rejected_KL_mean": -47.49944305419922, + "KL/std": 41.117652893066406, + "epoch": 0.2680628272251309, + "fcm_dpo/beta": 0.011355388909578323, + "fcm_dpo/delta": 0.0041136001236736774, + "fcm_dpo/margin": 30.299209594726562, + "fcm_dpo/q_t": 0.4208820164203644, + "grad_norm": 53.725135803222656, + "learning_rate": 4.5931796656116837e-07, + "logits/chosen": -0.7902308702468872, + "logits/rejected": -0.7869139909744263, + "logps/chosen": -282.59759521484375, + "logps/ref_chosen": -265.3973693847656, + "logps/ref_rejected": -250.9737548828125, + "logps/rejected": -298.47320556640625, + "loss": 4.5971, + "margin_dpo/margin_mean": 30.299209594726562, + "margin_dpo/margin_std": 49.29853820800781, + "step": 128 + }, + { + "KL/chosen_KL_mean": -18.242015838623047, + "KL/mean": -31.862220764160156, + "KL/rejected_KL_mean": -45.482425689697266, + "KL/std": 38.571449279785156, + "epoch": 0.27015706806282724, + "fcm_dpo/beta": 0.011395599693059921, + "fcm_dpo/delta": -0.014722984284162521, + "fcm_dpo/margin": 27.24040985107422, + "fcm_dpo/q_t": 0.4288609027862549, + "grad_norm": 56.34138488769531, + "learning_rate": 4.5831132482724193e-07, + "logits/chosen": -0.8255420327186584, + "logits/rejected": -0.8241854310035706, + "logps/chosen": -321.40045166015625, + "logps/ref_chosen": -303.158447265625, + "logps/ref_rejected": -275.9891052246094, + "logps/rejected": -321.47149658203125, + "loss": 4.7074, + "margin_dpo/margin_mean": 27.24040985107422, + "margin_dpo/margin_std": 48.436065673828125, + "step": 129 + }, + { + "KL/chosen_KL_mean": -20.89402961730957, + "KL/mean": -34.46953201293945, + "KL/rejected_KL_mean": -48.04502487182617, + "KL/std": 38.12776565551758, + "epoch": 0.27225130890052357, + "fcm_dpo/beta": 0.011203363537788391, + "fcm_dpo/delta": -0.00045868102461099625, + "fcm_dpo/margin": 27.150991439819336, + "fcm_dpo/q_t": 0.4299607276916504, + "grad_norm": 62.90643310546875, + "learning_rate": 4.5729351198915705e-07, + "logits/chosen": -0.7972059845924377, + "logits/rejected": -0.8352242112159729, + "logps/chosen": -307.3013610839844, + "logps/ref_chosen": -286.4073486328125, + "logps/ref_rejected": -294.38665771484375, + "logps/rejected": -342.4317321777344, + "loss": 4.7155, + "margin_dpo/margin_mean": 27.1509952545166, + "margin_dpo/margin_std": 48.800880432128906, + "step": 130 + }, + { + "KL/chosen_KL_mean": -25.511310577392578, + "KL/mean": -36.389923095703125, + "KL/rejected_KL_mean": -47.268531799316406, + "KL/std": 42.743492126464844, + "epoch": 0.2743455497382199, + "fcm_dpo/beta": 0.011295843869447708, + "fcm_dpo/delta": 0.008524668402969837, + "fcm_dpo/margin": 21.75722312927246, + "fcm_dpo/q_t": 0.4437254071235657, + "grad_norm": 66.18495178222656, + "learning_rate": 4.5626458262912735e-07, + "logits/chosen": -0.8689754009246826, + "logits/rejected": -0.8165445327758789, + "logps/chosen": -337.0763854980469, + "logps/ref_chosen": -311.5650634765625, + "logps/ref_rejected": -291.62432861328125, + "logps/rejected": -338.89288330078125, + "loss": 4.9757, + "margin_dpo/margin_mean": 21.75722312927246, + "margin_dpo/margin_std": 51.93965148925781, + "step": 131 + }, + { + "KL/chosen_KL_mean": -21.638566970825195, + "KL/mean": -36.89574432373047, + "KL/rejected_KL_mean": -52.152923583984375, + "KL/std": 42.402366638183594, + "epoch": 0.2764397905759162, + "fcm_dpo/beta": 0.01130670215934515, + "fcm_dpo/delta": -0.03159793093800545, + "fcm_dpo/margin": 30.514366149902344, + "fcm_dpo/q_t": 0.42199385166168213, + "grad_norm": 70.13489532470703, + "learning_rate": 4.5522459192551166e-07, + "logits/chosen": -0.8335902690887451, + "logits/rejected": -0.8173937797546387, + "logps/chosen": -291.72039794921875, + "logps/ref_chosen": -270.0818176269531, + "logps/ref_rejected": -284.3084411621094, + "logps/rejected": -336.46136474609375, + "loss": 4.658, + "margin_dpo/margin_mean": 30.514366149902344, + "margin_dpo/margin_std": 53.337528228759766, + "step": 132 + }, + { + "KL/chosen_KL_mean": -24.91779327392578, + "KL/mean": -39.1947135925293, + "KL/rejected_KL_mean": -53.47163009643555, + "KL/std": 39.884578704833984, + "epoch": 0.27853403141361255, + "fcm_dpo/beta": 0.011096285656094551, + "fcm_dpo/delta": -0.002665869891643524, + "fcm_dpo/margin": 28.553842544555664, + "fcm_dpo/q_t": 0.42722228169441223, + "grad_norm": 47.607017517089844, + "learning_rate": 4.541735956498554e-07, + "logits/chosen": -0.8624144196510315, + "logits/rejected": -0.8618667721748352, + "logps/chosen": -310.5391540527344, + "logps/ref_chosen": -285.6213684082031, + "logps/ref_rejected": -251.19386291503906, + "logps/rejected": -304.66546630859375, + "loss": 4.6841, + "margin_dpo/margin_mean": 28.553842544555664, + "margin_dpo/margin_std": 49.61962127685547, + "step": 133 + }, + { + "KL/chosen_KL_mean": -37.28353500366211, + "KL/mean": -47.89252471923828, + "KL/rejected_KL_mean": -58.50152587890625, + "KL/std": 41.15799331665039, + "epoch": 0.2806282722513089, + "fcm_dpo/beta": 0.010984513908624649, + "fcm_dpo/delta": -0.00815630704164505, + "fcm_dpo/margin": 21.21798324584961, + "fcm_dpo/q_t": 0.4452402591705322, + "grad_norm": 54.9819450378418, + "learning_rate": 4.5311165016389914e-07, + "logits/chosen": -0.8601773977279663, + "logits/rejected": -0.8593323230743408, + "logps/chosen": -356.204345703125, + "logps/ref_chosen": -318.92083740234375, + "logps/ref_rejected": -293.1894836425781, + "logps/rejected": -351.69097900390625, + "loss": 4.9206, + "margin_dpo/margin_mean": 21.21798324584961, + "margin_dpo/margin_std": 44.41059875488281, + "step": 134 + }, + { + "KL/chosen_KL_mean": -34.964073181152344, + "KL/mean": -47.551513671875, + "KL/rejected_KL_mean": -60.138946533203125, + "KL/std": 43.32963943481445, + "epoch": 0.28272251308900526, + "fcm_dpo/beta": 0.011102442629635334, + "fcm_dpo/delta": 0.0017053596675395966, + "fcm_dpo/margin": 25.174869537353516, + "fcm_dpo/q_t": 0.4345686733722687, + "grad_norm": 65.74720001220703, + "learning_rate": 4.520388124165564e-07, + "logits/chosen": -0.7585154175758362, + "logits/rejected": -0.798126757144928, + "logps/chosen": -327.7857971191406, + "logps/ref_chosen": -292.8217468261719, + "logps/ref_rejected": -269.2896728515625, + "logps/rejected": -329.42864990234375, + "loss": 4.8261, + "margin_dpo/margin_mean": 25.174869537353516, + "margin_dpo/margin_std": 48.767791748046875, + "step": 135 + }, + { + "KL/chosen_KL_mean": -37.39518356323242, + "KL/mean": -51.40061950683594, + "KL/rejected_KL_mean": -65.40606689453125, + "KL/std": 43.074127197265625, + "epoch": 0.2848167539267016, + "fcm_dpo/beta": 0.010944095440208912, + "fcm_dpo/delta": -0.015430422499775887, + "fcm_dpo/margin": 28.01087760925293, + "fcm_dpo/q_t": 0.4303656220436096, + "grad_norm": 75.27836608886719, + "learning_rate": 4.5095513994085974e-07, + "logits/chosen": -0.8118724226951599, + "logits/rejected": -0.806470513343811, + "logps/chosen": -310.2476806640625, + "logps/ref_chosen": -272.8525390625, + "logps/ref_rejected": -252.68202209472656, + "logps/rejected": -318.08807373046875, + "loss": 4.7693, + "margin_dpo/margin_mean": 28.01087760925293, + "margin_dpo/margin_std": 54.0770378112793, + "step": 136 + }, + { + "KL/chosen_KL_mean": -41.546897888183594, + "KL/mean": -54.02006149291992, + "KL/rejected_KL_mean": -66.49323272705078, + "KL/std": 45.12068176269531, + "epoch": 0.2869109947643979, + "fcm_dpo/beta": 0.010924424976110458, + "fcm_dpo/delta": 0.014189784415066242, + "fcm_dpo/margin": 24.94634437561035, + "fcm_dpo/q_t": 0.43837597966194153, + "grad_norm": 66.66802978515625, + "learning_rate": 4.498606908508753e-07, + "logits/chosen": -0.859586775302887, + "logits/rejected": -0.8458654880523682, + "logps/chosen": -342.2991027832031, + "logps/ref_chosen": -300.7522277832031, + "logps/ref_rejected": -286.1935119628906, + "logps/rejected": -352.686767578125, + "loss": 4.8497, + "margin_dpo/margin_mean": 24.94634437561035, + "margin_dpo/margin_std": 51.542877197265625, + "step": 137 + }, + { + "KL/chosen_KL_mean": -38.696102142333984, + "KL/mean": -53.19694900512695, + "KL/rejected_KL_mean": -67.69778442382812, + "KL/std": 46.134708404541016, + "epoch": 0.28900523560209423, + "fcm_dpo/beta": 0.011035628616809845, + "fcm_dpo/delta": 0.011557774618268013, + "fcm_dpo/margin": 29.00168228149414, + "fcm_dpo/q_t": 0.42893779277801514, + "grad_norm": 61.13792419433594, + "learning_rate": 4.487555238385862e-07, + "logits/chosen": -0.7798894047737122, + "logits/rejected": -0.7592524290084839, + "logps/chosen": -327.63311767578125, + "logps/ref_chosen": -288.9369812011719, + "logps/ref_rejected": -263.7076416015625, + "logps/rejected": -331.40545654296875, + "loss": 4.7942, + "margin_dpo/margin_mean": 29.00168228149414, + "margin_dpo/margin_std": 60.18000030517578, + "step": 138 + }, + { + "KL/chosen_KL_mean": -35.5364990234375, + "KL/mean": -45.61208724975586, + "KL/rejected_KL_mean": -55.68767547607422, + "KL/std": 41.366214752197266, + "epoch": 0.29109947643979056, + "fcm_dpo/beta": 0.01104138232767582, + "fcm_dpo/delta": -0.008695616386830807, + "fcm_dpo/margin": 20.151180267333984, + "fcm_dpo/q_t": 0.4494304060935974, + "grad_norm": 69.4866714477539, + "learning_rate": 4.476396981707453e-07, + "logits/chosen": -0.8099366426467896, + "logits/rejected": -0.8398616313934326, + "logps/chosen": -305.580810546875, + "logps/ref_chosen": -270.0443115234375, + "logps/ref_rejected": -267.3226013183594, + "logps/rejected": -323.01025390625, + "loss": 5.0408, + "margin_dpo/margin_mean": 20.151180267333984, + "margin_dpo/margin_std": 51.15916442871094, + "step": 139 + }, + { + "KL/chosen_KL_mean": -33.201412200927734, + "KL/mean": -48.6481819152832, + "KL/rejected_KL_mean": -64.09495544433594, + "KL/std": 43.8369255065918, + "epoch": 0.2931937172774869, + "fcm_dpo/beta": 0.010931570082902908, + "fcm_dpo/delta": -0.034940119832754135, + "fcm_dpo/margin": 30.893543243408203, + "fcm_dpo/q_t": 0.4216638505458832, + "grad_norm": 62.90419006347656, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": -0.8739014863967896, + "logits/rejected": -0.8388884663581848, + "logps/chosen": -316.156982421875, + "logps/ref_chosen": -282.9555969238281, + "logps/ref_rejected": -251.17181396484375, + "logps/rejected": -315.2667541503906, + "loss": 4.596, + "margin_dpo/margin_mean": 30.893543243408203, + "margin_dpo/margin_std": 47.828704833984375, + "step": 140 + }, + { + "KL/chosen_KL_mean": -31.26448631286621, + "KL/mean": -47.31875228881836, + "KL/rejected_KL_mean": -63.37302017211914, + "KL/std": 45.12312316894531, + "epoch": 0.29528795811518327, + "fcm_dpo/beta": 0.010553342290222645, + "fcm_dpo/delta": -0.02428608201444149, + "fcm_dpo/margin": 32.10853958129883, + "fcm_dpo/q_t": 0.42113417387008667, + "grad_norm": 68.2318344116211, + "learning_rate": 4.453763107901675e-07, + "logits/chosen": -0.8127657771110535, + "logits/rejected": -0.8073655366897583, + "logps/chosen": -327.56463623046875, + "logps/ref_chosen": -296.3001708984375, + "logps/ref_rejected": -279.8486633300781, + "logps/rejected": -343.2216796875, + "loss": 4.6544, + "margin_dpo/margin_mean": 32.10853958129883, + "margin_dpo/margin_std": 55.96519470214844, + "step": 141 + }, + { + "KL/chosen_KL_mean": -32.01213836669922, + "KL/mean": -45.19526672363281, + "KL/rejected_KL_mean": -58.378387451171875, + "KL/std": 45.81889343261719, + "epoch": 0.2973821989528796, + "fcm_dpo/beta": 0.01045268028974533, + "fcm_dpo/delta": -0.0036399501841515303, + "fcm_dpo/margin": 26.366239547729492, + "fcm_dpo/q_t": 0.4370165467262268, + "grad_norm": 58.871788024902344, + "learning_rate": 4.4422887045602674e-07, + "logits/chosen": -0.8336638808250427, + "logits/rejected": -0.8351055979728699, + "logps/chosen": -332.5780029296875, + "logps/ref_chosen": -300.56585693359375, + "logps/ref_rejected": -231.43316650390625, + "logps/rejected": -289.8115539550781, + "loss": 4.8816, + "margin_dpo/margin_mean": 26.366239547729492, + "margin_dpo/margin_std": 56.89813995361328, + "step": 142 + }, + { + "KL/chosen_KL_mean": -31.096927642822266, + "KL/mean": -47.20490264892578, + "KL/rejected_KL_mean": -63.3128776550293, + "KL/std": 43.217559814453125, + "epoch": 0.2994764397905759, + "fcm_dpo/beta": 0.01030801422894001, + "fcm_dpo/delta": -0.026560742408037186, + "fcm_dpo/margin": 32.2159538269043, + "fcm_dpo/q_t": 0.42342373728752136, + "grad_norm": 58.806697845458984, + "learning_rate": 4.4307101421701755e-07, + "logits/chosen": -0.8223780393600464, + "logits/rejected": -0.8012036681175232, + "logps/chosen": -327.82928466796875, + "logps/ref_chosen": -296.73236083984375, + "logps/ref_rejected": -266.45257568359375, + "logps/rejected": -329.7654724121094, + "loss": 4.6057, + "margin_dpo/margin_mean": 32.2159538269043, + "margin_dpo/margin_std": 50.72016143798828, + "step": 143 + }, + { + "KL/chosen_KL_mean": -31.219467163085938, + "KL/mean": -44.27275848388672, + "KL/rejected_KL_mean": -57.326053619384766, + "KL/std": 46.5449333190918, + "epoch": 0.30157068062827225, + "fcm_dpo/beta": 0.010053442791104317, + "fcm_dpo/delta": -0.015576949343085289, + "fcm_dpo/margin": 26.106597900390625, + "fcm_dpo/q_t": 0.44039374589920044, + "grad_norm": 52.784488677978516, + "learning_rate": 4.419028041654559e-07, + "logits/chosen": -0.8913372755050659, + "logits/rejected": -0.8779230117797852, + "logps/chosen": -330.06341552734375, + "logps/ref_chosen": -298.843994140625, + "logps/ref_rejected": -266.120849609375, + "logps/rejected": -323.4468994140625, + "loss": 4.8861, + "margin_dpo/margin_mean": 26.10660171508789, + "margin_dpo/margin_std": 56.03483963012695, + "step": 144 + }, + { + "KL/chosen_KL_mean": -27.039196014404297, + "KL/mean": -43.98484420776367, + "KL/rejected_KL_mean": -60.930477142333984, + "KL/std": 42.91300964355469, + "epoch": 0.3036649214659686, + "fcm_dpo/beta": 0.00988059677183628, + "fcm_dpo/delta": -0.03598638251423836, + "fcm_dpo/margin": 33.89128112792969, + "fcm_dpo/q_t": 0.42246097326278687, + "grad_norm": 58.97926712036133, + "learning_rate": 4.4072430294890166e-07, + "logits/chosen": -0.8730674982070923, + "logits/rejected": -0.8762320876121521, + "logps/chosen": -302.79205322265625, + "logps/ref_chosen": -275.7528381347656, + "logps/ref_rejected": -214.74807739257812, + "logps/rejected": -275.6785583496094, + "loss": 4.6006, + "margin_dpo/margin_mean": 33.89128112792969, + "margin_dpo/margin_std": 52.56437683105469, + "step": 145 + }, + { + "KL/chosen_KL_mean": -36.02202606201172, + "KL/mean": -52.752845764160156, + "KL/rejected_KL_mean": -69.48365783691406, + "KL/std": 47.3927001953125, + "epoch": 0.3057591623036649, + "fcm_dpo/beta": 0.009669817984104156, + "fcm_dpo/delta": -0.026783179491758347, + "fcm_dpo/margin": 33.461639404296875, + "fcm_dpo/q_t": 0.42534470558166504, + "grad_norm": 59.773223876953125, + "learning_rate": 4.395355737667985e-07, + "logits/chosen": -0.8538410067558289, + "logits/rejected": -0.848624587059021, + "logps/chosen": -313.1202392578125, + "logps/ref_chosen": -277.09820556640625, + "logps/ref_rejected": -265.41046142578125, + "logps/rejected": -334.89410400390625, + "loss": 4.6348, + "margin_dpo/margin_mean": 33.461639404296875, + "margin_dpo/margin_std": 52.627410888671875, + "step": 146 + }, + { + "KL/chosen_KL_mean": -37.466670989990234, + "KL/mean": -50.719993591308594, + "KL/rejected_KL_mean": -63.97330856323242, + "KL/std": 49.072593688964844, + "epoch": 0.3078534031413613, + "fcm_dpo/beta": 0.009586220607161522, + "fcm_dpo/delta": -0.023667726665735245, + "fcm_dpo/margin": 26.50663185119629, + "fcm_dpo/q_t": 0.4408421814441681, + "grad_norm": 54.904624938964844, + "learning_rate": 4.3833668036708483e-07, + "logits/chosen": -0.83968186378479, + "logits/rejected": -0.8350346088409424, + "logps/chosen": -328.88525390625, + "logps/ref_chosen": -291.4185791015625, + "logps/ref_rejected": -253.43051147460938, + "logps/rejected": -317.40380859375, + "loss": 4.9045, + "margin_dpo/margin_mean": 26.506629943847656, + "margin_dpo/margin_std": 54.914512634277344, + "step": 147 + }, + { + "KL/chosen_KL_mean": -35.33280944824219, + "KL/mean": -49.60417175292969, + "KL/rejected_KL_mean": -63.87554168701172, + "KL/std": 51.94282913208008, + "epoch": 0.3099476439790576, + "fcm_dpo/beta": 0.009600591845810413, + "fcm_dpo/delta": 0.039563555270433426, + "fcm_dpo/margin": 28.542724609375, + "fcm_dpo/q_t": 0.43925410509109497, + "grad_norm": 56.438453674316406, + "learning_rate": 4.3712768704277524e-07, + "logits/chosen": -0.9007651209831238, + "logits/rejected": -0.9030435681343079, + "logps/chosen": -272.0813293457031, + "logps/ref_chosen": -236.74850463867188, + "logps/ref_rejected": -231.4674072265625, + "logps/rejected": -295.3429260253906, + "loss": 4.8703, + "margin_dpo/margin_mean": 28.542724609375, + "margin_dpo/margin_std": 62.54542922973633, + "step": 148 + }, + { + "KL/chosen_KL_mean": -41.490516662597656, + "KL/mean": -60.27011489868164, + "KL/rejected_KL_mean": -79.04971313476562, + "KL/std": 47.53480911254883, + "epoch": 0.31204188481675393, + "fcm_dpo/beta": 0.009786082431674004, + "fcm_dpo/delta": 0.0004139197990298271, + "fcm_dpo/margin": 37.55918884277344, + "fcm_dpo/q_t": 0.4152625799179077, + "grad_norm": 60.076107025146484, + "learning_rate": 4.3590865862851263e-07, + "logits/chosen": -0.8776776790618896, + "logits/rejected": -0.8625072836875916, + "logps/chosen": -361.41900634765625, + "logps/ref_chosen": -319.9284973144531, + "logps/ref_rejected": -308.20233154296875, + "logps/rejected": -387.2520751953125, + "loss": 4.4703, + "margin_dpo/margin_mean": 37.55918884277344, + "margin_dpo/margin_std": 52.95542526245117, + "step": 149 + }, + { + "KL/chosen_KL_mean": -47.29694366455078, + "KL/mean": -62.3214225769043, + "KL/rejected_KL_mean": -77.34589385986328, + "KL/std": 50.34033966064453, + "epoch": 0.31413612565445026, + "fcm_dpo/beta": 0.009834048338234425, + "fcm_dpo/delta": 0.013316557742655277, + "fcm_dpo/margin": 30.048938751220703, + "fcm_dpo/q_t": 0.4316878020763397, + "grad_norm": 59.99736022949219, + "learning_rate": 4.346796604970912e-07, + "logits/chosen": -0.8376021981239319, + "logits/rejected": -0.8213092088699341, + "logps/chosen": -323.61517333984375, + "logps/ref_chosen": -276.3182373046875, + "logps/ref_rejected": -273.02215576171875, + "logps/rejected": -350.3680114746094, + "loss": 4.7488, + "margin_dpo/margin_mean": 30.04894256591797, + "margin_dpo/margin_std": 55.58155059814453, + "step": 150 + }, + { + "KL/chosen_KL_mean": -44.79521560668945, + "KL/mean": -66.85020446777344, + "KL/rejected_KL_mean": -88.90518951416016, + "KL/std": 50.85723876953125, + "epoch": 0.3162303664921466, + "fcm_dpo/beta": 0.009750737808644772, + "fcm_dpo/delta": -0.03168368339538574, + "fcm_dpo/margin": 44.109981536865234, + "fcm_dpo/q_t": 0.4018837809562683, + "grad_norm": 62.32393264770508, + "learning_rate": 4.3344075855595097e-07, + "logits/chosen": -0.8728746771812439, + "logits/rejected": -0.8648634552955627, + "logps/chosen": -342.1080322265625, + "logps/ref_chosen": -297.31280517578125, + "logps/ref_rejected": -266.1003723144531, + "logps/rejected": -355.0055847167969, + "loss": 4.3372, + "margin_dpo/margin_mean": 44.109981536865234, + "margin_dpo/margin_std": 58.83431625366211, + "step": 151 + }, + { + "KL/chosen_KL_mean": -39.10889434814453, + "KL/mean": -57.962406158447266, + "KL/rejected_KL_mean": -76.81593322753906, + "KL/std": 53.661624908447266, + "epoch": 0.3183246073298429, + "fcm_dpo/beta": 0.009311305359005928, + "fcm_dpo/delta": -0.05312967672944069, + "fcm_dpo/margin": 37.7070198059082, + "fcm_dpo/q_t": 0.421106219291687, + "grad_norm": 54.742496490478516, + "learning_rate": 4.3219201924364323e-07, + "logits/chosen": -0.8810481429100037, + "logits/rejected": -0.8775116205215454, + "logps/chosen": -309.35589599609375, + "logps/ref_chosen": -270.2470397949219, + "logps/ref_rejected": -269.7749328613281, + "logps/rejected": -346.59088134765625, + "loss": 4.6232, + "margin_dpo/margin_mean": 37.7070198059082, + "margin_dpo/margin_std": 62.859439849853516, + "step": 152 + }, + { + "KL/chosen_KL_mean": -46.92686462402344, + "KL/mean": -73.7924575805664, + "KL/rejected_KL_mean": -100.65806579589844, + "KL/std": 58.3165397644043, + "epoch": 0.3204188481675393, + "fcm_dpo/beta": 0.008776971139013767, + "fcm_dpo/delta": -0.09027046710252762, + "fcm_dpo/margin": 53.731204986572266, + "fcm_dpo/q_t": 0.39192867279052734, + "grad_norm": 56.017066955566406, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": -0.8752989172935486, + "logits/rejected": -0.8623684048652649, + "logps/chosen": -320.7059326171875, + "logps/ref_chosen": -273.779052734375, + "logps/ref_rejected": -280.9530944824219, + "logps/rejected": -381.6112060546875, + "loss": 4.1495, + "margin_dpo/margin_mean": 53.731204986572266, + "margin_dpo/margin_std": 60.12602996826172, + "step": 153 + }, + { + "KL/chosen_KL_mean": -47.703121185302734, + "KL/mean": -68.70870208740234, + "KL/rejected_KL_mean": -89.71429443359375, + "KL/std": 60.09400177001953, + "epoch": 0.3225130890052356, + "fcm_dpo/beta": 0.0084590008482337, + "fcm_dpo/delta": -0.023284776136279106, + "fcm_dpo/margin": 42.011173248291016, + "fcm_dpo/q_t": 0.41954493522644043, + "grad_norm": 59.05760955810547, + "learning_rate": 4.2966529689388064e-07, + "logits/chosen": -0.9124890565872192, + "logits/rejected": -0.8928197622299194, + "logps/chosen": -337.6063232421875, + "logps/ref_chosen": -289.9031982421875, + "logps/ref_rejected": -261.5166320800781, + "logps/rejected": -351.2309265136719, + "loss": 4.6048, + "margin_dpo/margin_mean": 42.011173248291016, + "margin_dpo/margin_std": 70.03306579589844, + "step": 154 + }, + { + "KL/chosen_KL_mean": -54.11407470703125, + "KL/mean": -72.34459686279297, + "KL/rejected_KL_mean": -90.57511901855469, + "KL/std": 57.328651428222656, + "epoch": 0.32460732984293195, + "fcm_dpo/beta": 0.008467341773211956, + "fcm_dpo/delta": 0.010241080075502396, + "fcm_dpo/margin": 36.461036682128906, + "fcm_dpo/q_t": 0.4311079978942871, + "grad_norm": 69.09295654296875, + "learning_rate": 4.2838744935687716e-07, + "logits/chosen": -0.8517003655433655, + "logits/rejected": -0.848478376865387, + "logps/chosen": -339.97528076171875, + "logps/ref_chosen": -285.8612060546875, + "logps/ref_rejected": -300.1272888183594, + "logps/rejected": -390.702392578125, + "loss": 4.7183, + "margin_dpo/margin_mean": 36.461036682128906, + "margin_dpo/margin_std": 65.83427429199219, + "step": 155 + }, + { + "KL/chosen_KL_mean": -50.88059997558594, + "KL/mean": -76.42092895507812, + "KL/rejected_KL_mean": -101.96127319335938, + "KL/std": 61.5806884765625, + "epoch": 0.3267015706806283, + "fcm_dpo/beta": 0.008312368765473366, + "fcm_dpo/delta": -0.07811664044857025, + "fcm_dpo/margin": 51.08067321777344, + "fcm_dpo/q_t": 0.4043055474758148, + "grad_norm": 58.513641357421875, + "learning_rate": 4.271000354423425e-07, + "logits/chosen": -0.8742014765739441, + "logits/rejected": -0.8701291084289551, + "logps/chosen": -329.91607666015625, + "logps/ref_chosen": -279.0354919433594, + "logps/ref_rejected": -244.2198486328125, + "logps/rejected": -346.1811218261719, + "loss": 4.4398, + "margin_dpo/margin_mean": 51.08067321777344, + "margin_dpo/margin_std": 77.06956481933594, + "step": 156 + }, + { + "KL/chosen_KL_mean": -54.794883728027344, + "KL/mean": -74.77871704101562, + "KL/rejected_KL_mean": -94.76253509521484, + "KL/std": 61.28529357910156, + "epoch": 0.3287958115183246, + "fcm_dpo/beta": 0.007772133685648441, + "fcm_dpo/delta": -0.0433635450899601, + "fcm_dpo/margin": 39.9676513671875, + "fcm_dpo/q_t": 0.4275224804878235, + "grad_norm": 53.20646286010742, + "learning_rate": 4.258031241903777e-07, + "logits/chosen": -0.9390982985496521, + "logits/rejected": -0.937108039855957, + "logps/chosen": -325.62518310546875, + "logps/ref_chosen": -270.830322265625, + "logps/ref_rejected": -259.08319091796875, + "logps/rejected": -353.84576416015625, + "loss": 4.6787, + "margin_dpo/margin_mean": 39.9676513671875, + "margin_dpo/margin_std": 66.26377868652344, + "step": 157 + }, + { + "KL/chosen_KL_mean": -54.49103546142578, + "KL/mean": -77.38906860351562, + "KL/rejected_KL_mean": -100.28709411621094, + "KL/std": 62.60470199584961, + "epoch": 0.3308900523560209, + "fcm_dpo/beta": 0.007661835290491581, + "fcm_dpo/delta": -0.022122088819742203, + "fcm_dpo/margin": 45.79604721069336, + "fcm_dpo/q_t": 0.4180770218372345, + "grad_norm": 55.99773406982422, + "learning_rate": 4.2449678515039743e-07, + "logits/chosen": -0.8606072068214417, + "logits/rejected": -0.8443244695663452, + "logps/chosen": -344.4573669433594, + "logps/ref_chosen": -289.9663391113281, + "logps/ref_rejected": -271.335693359375, + "logps/rejected": -371.6227722167969, + "loss": 4.5509, + "margin_dpo/margin_mean": 45.796051025390625, + "margin_dpo/margin_std": 67.24652862548828, + "step": 158 + }, + { + "KL/chosen_KL_mean": -60.751251220703125, + "KL/mean": -74.46627807617188, + "KL/rejected_KL_mean": -88.18130493164062, + "KL/std": 58.07416534423828, + "epoch": 0.33298429319371725, + "fcm_dpo/beta": 0.007585161365568638, + "fcm_dpo/delta": 0.009133652783930302, + "fcm_dpo/margin": 27.430057525634766, + "fcm_dpo/q_t": 0.4531330466270447, + "grad_norm": 74.61146545410156, + "learning_rate": 4.2318108837739986e-07, + "logits/chosen": -0.9520941972732544, + "logits/rejected": -0.9025680422782898, + "logps/chosen": -382.12957763671875, + "logps/ref_chosen": -321.37835693359375, + "logps/ref_rejected": -250.45652770996094, + "logps/rejected": -338.63787841796875, + "loss": 5.0799, + "margin_dpo/margin_mean": 27.430057525634766, + "margin_dpo/margin_std": 72.122802734375, + "step": 159 + }, + { + "KL/chosen_KL_mean": -50.046653747558594, + "KL/mean": -75.42059326171875, + "KL/rejected_KL_mean": -100.79450988769531, + "KL/std": 61.544471740722656, + "epoch": 0.33507853403141363, + "fcm_dpo/beta": 0.00758227426558733, + "fcm_dpo/delta": -0.000992432702332735, + "fcm_dpo/margin": 50.74785614013672, + "fcm_dpo/q_t": 0.41024836897850037, + "grad_norm": 55.165855407714844, + "learning_rate": 4.218561044282098e-07, + "logits/chosen": -0.8842563629150391, + "logits/rejected": -0.8855242133140564, + "logps/chosen": -326.3301696777344, + "logps/ref_chosen": -276.28350830078125, + "logps/ref_rejected": -262.7477722167969, + "logps/rejected": -363.54229736328125, + "loss": 4.3613, + "margin_dpo/margin_mean": 50.74785614013672, + "margin_dpo/margin_std": 59.35392761230469, + "step": 160 + }, + { + "KL/chosen_KL_mean": -59.74705123901367, + "KL/mean": -82.03538513183594, + "KL/rejected_KL_mean": -104.32372283935547, + "KL/std": 64.49849700927734, + "epoch": 0.33717277486910996, + "fcm_dpo/beta": 0.00754944933578372, + "fcm_dpo/delta": -0.02097606658935547, + "fcm_dpo/margin": 44.5766716003418, + "fcm_dpo/q_t": 0.42417094111442566, + "grad_norm": 75.2176284790039, + "learning_rate": 4.2052190435769554e-07, + "logits/chosen": -0.8945316076278687, + "logits/rejected": -0.8761196136474609, + "logps/chosen": -370.2398376464844, + "logps/ref_chosen": -310.4927978515625, + "logps/ref_rejected": -250.25347900390625, + "logps/rejected": -354.57720947265625, + "loss": 4.6438, + "margin_dpo/margin_mean": 44.5766716003418, + "margin_dpo/margin_std": 75.5418701171875, + "step": 161 + }, + { + "KL/chosen_KL_mean": -54.763946533203125, + "KL/mean": -75.6114273071289, + "KL/rejected_KL_mean": -96.45890808105469, + "KL/std": 60.076480865478516, + "epoch": 0.3392670157068063, + "fcm_dpo/beta": 0.00734911672770977, + "fcm_dpo/delta": -0.007708599790930748, + "fcm_dpo/margin": 41.69496154785156, + "fcm_dpo/q_t": 0.42868733406066895, + "grad_norm": 45.79068374633789, + "learning_rate": 4.1917855971495763e-07, + "logits/chosen": -0.8724191784858704, + "logits/rejected": -0.8640581965446472, + "logps/chosen": -350.8743896484375, + "logps/ref_chosen": -296.1105041503906, + "logps/ref_rejected": -253.4247589111328, + "logps/rejected": -349.8836669921875, + "loss": 4.6753, + "margin_dpo/margin_mean": 41.69496536254883, + "margin_dpo/margin_std": 67.41392517089844, + "step": 162 + }, + { + "KL/chosen_KL_mean": -59.032291412353516, + "KL/mean": -82.56084442138672, + "KL/rejected_KL_mean": -106.08937072753906, + "KL/std": 59.548545837402344, + "epoch": 0.3413612565445026, + "fcm_dpo/beta": 0.00737118162214756, + "fcm_dpo/delta": -0.02219436690211296, + "fcm_dpo/margin": 47.05707550048828, + "fcm_dpo/q_t": 0.4185625910758972, + "grad_norm": 65.94065856933594, + "learning_rate": 4.1782614253949255e-07, + "logits/chosen": -0.9133932590484619, + "logits/rejected": -0.9096333384513855, + "logps/chosen": -352.5321960449219, + "logps/ref_chosen": -293.4999084472656, + "logps/ref_rejected": -266.7116394042969, + "logps/rejected": -372.801025390625, + "loss": 4.5254, + "margin_dpo/margin_mean": 47.05707550048828, + "margin_dpo/margin_std": 65.4988021850586, + "step": 163 + }, + { + "KL/chosen_KL_mean": -69.27760314941406, + "KL/mean": -90.69017028808594, + "KL/rejected_KL_mean": -112.10274505615234, + "KL/std": 66.29005432128906, + "epoch": 0.34345549738219894, + "fcm_dpo/beta": 0.007323879282921553, + "fcm_dpo/delta": -0.013420582748949528, + "fcm_dpo/margin": 42.82514572143555, + "fcm_dpo/q_t": 0.42865079641342163, + "grad_norm": 72.55785369873047, + "learning_rate": 4.164647253573289e-07, + "logits/chosen": -0.86087965965271, + "logits/rejected": -0.8710927963256836, + "logps/chosen": -336.3271179199219, + "logps/ref_chosen": -267.04949951171875, + "logps/ref_rejected": -215.9768829345703, + "logps/rejected": -328.0796203613281, + "loss": 4.7004, + "margin_dpo/margin_mean": 42.82514572143555, + "margin_dpo/margin_std": 75.33882904052734, + "step": 164 + }, + { + "KL/chosen_KL_mean": -57.162994384765625, + "KL/mean": -72.81074523925781, + "KL/rejected_KL_mean": -88.45850372314453, + "KL/std": 55.15323257446289, + "epoch": 0.34554973821989526, + "fcm_dpo/beta": 0.007216047961264849, + "fcm_dpo/delta": 0.00300390413030982, + "fcm_dpo/margin": 31.295488357543945, + "fcm_dpo/q_t": 0.4470052123069763, + "grad_norm": 65.5068588256836, + "learning_rate": 4.1509438117713863e-07, + "logits/chosen": -0.8794437646865845, + "logits/rejected": -0.8518754243850708, + "logps/chosen": -335.2244873046875, + "logps/ref_chosen": -278.06146240234375, + "logps/ref_rejected": -260.4288635253906, + "logps/rejected": -348.88739013671875, + "loss": 4.908, + "margin_dpo/margin_mean": 31.295486450195312, + "margin_dpo/margin_std": 61.843997955322266, + "step": 165 + }, + { + "KL/chosen_KL_mean": -51.96410369873047, + "KL/mean": -71.39328002929688, + "KL/rejected_KL_mean": -90.82244873046875, + "KL/std": 58.12259292602539, + "epoch": 0.34764397905759165, + "fcm_dpo/beta": 0.007127226330339909, + "fcm_dpo/delta": -0.0192001573741436, + "fcm_dpo/margin": 38.85832977294922, + "fcm_dpo/q_t": 0.43620848655700684, + "grad_norm": 58.496612548828125, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": -0.8499168157577515, + "logits/rejected": -0.8151109218597412, + "logps/chosen": -327.91314697265625, + "logps/ref_chosen": -275.9490661621094, + "logps/ref_rejected": -232.13473510742188, + "logps/rejected": -322.9571838378906, + "loss": 4.776, + "margin_dpo/margin_mean": 38.85832977294922, + "margin_dpo/margin_std": 71.73967742919922, + "step": 166 + }, + { + "KL/chosen_KL_mean": -60.47264099121094, + "KL/mean": -83.17312622070312, + "KL/rejected_KL_mean": -105.87361145019531, + "KL/std": 63.884830474853516, + "epoch": 0.34973821989528797, + "fcm_dpo/beta": 0.007206224836409092, + "fcm_dpo/delta": 0.01624479703605175, + "fcm_dpo/margin": 45.400970458984375, + "fcm_dpo/q_t": 0.4238249361515045, + "grad_norm": 78.14469146728516, + "learning_rate": 4.123272062470633e-07, + "logits/chosen": -0.8364863395690918, + "logits/rejected": -0.8216645121574402, + "logps/chosen": -341.0240478515625, + "logps/ref_chosen": -280.5514221191406, + "logps/ref_rejected": -255.2896728515625, + "logps/rejected": -361.1632995605469, + "loss": 4.6281, + "margin_dpo/margin_mean": 45.400970458984375, + "margin_dpo/margin_std": 73.75985717773438, + "step": 167 + }, + { + "KL/chosen_KL_mean": -62.252197265625, + "KL/mean": -86.59355163574219, + "KL/rejected_KL_mean": -110.93489837646484, + "KL/std": 66.37821960449219, + "epoch": 0.3518324607329843, + "fcm_dpo/beta": 0.007193275727331638, + "fcm_dpo/delta": -0.006395729724317789, + "fcm_dpo/margin": 48.682701110839844, + "fcm_dpo/q_t": 0.41802850365638733, + "grad_norm": 124.62866973876953, + "learning_rate": 4.1093052389237174e-07, + "logits/chosen": -0.8164465427398682, + "logits/rejected": -0.7877838611602783, + "logps/chosen": -378.0504150390625, + "logps/ref_chosen": -315.7982177734375, + "logps/ref_rejected": -291.48406982421875, + "logps/rejected": -402.4189758300781, + "loss": 4.5506, + "margin_dpo/margin_mean": 48.682701110839844, + "margin_dpo/margin_std": 73.01609802246094, + "step": 168 + }, + { + "KL/chosen_KL_mean": -57.24802017211914, + "KL/mean": -87.19869232177734, + "KL/rejected_KL_mean": -117.14934539794922, + "KL/std": 67.05406188964844, + "epoch": 0.3539267015706806, + "fcm_dpo/beta": 0.007014545612037182, + "fcm_dpo/delta": -0.061939746141433716, + "fcm_dpo/margin": 59.90133285522461, + "fcm_dpo/q_t": 0.40285056829452515, + "grad_norm": 89.98390197753906, + "learning_rate": 4.0952521132208267e-07, + "logits/chosen": -0.8162607550621033, + "logits/rejected": -0.8252590894699097, + "logps/chosen": -318.3122863769531, + "logps/ref_chosen": -261.06427001953125, + "logps/ref_rejected": -235.40663146972656, + "logps/rejected": -352.55596923828125, + "loss": 4.3106, + "margin_dpo/margin_mean": 59.90133285522461, + "margin_dpo/margin_std": 72.81208801269531, + "step": 169 + }, + { + "KL/chosen_KL_mean": -56.95838165283203, + "KL/mean": -81.45191955566406, + "KL/rejected_KL_mean": -105.94544982910156, + "KL/std": 70.3679428100586, + "epoch": 0.35602094240837695, + "fcm_dpo/beta": 0.006829574704170227, + "fcm_dpo/delta": 0.006389938294887543, + "fcm_dpo/margin": 48.9870719909668, + "fcm_dpo/q_t": 0.42464104294776917, + "grad_norm": 79.10480499267578, + "learning_rate": 4.081113438988443e-07, + "logits/chosen": -0.7996067404747009, + "logits/rejected": -0.8019770383834839, + "logps/chosen": -365.9256286621094, + "logps/ref_chosen": -308.96722412109375, + "logps/ref_rejected": -263.8466796875, + "logps/rejected": -369.7921142578125, + "loss": 4.6634, + "margin_dpo/margin_mean": 48.98707580566406, + "margin_dpo/margin_std": 88.27198791503906, + "step": 170 + }, + { + "KL/chosen_KL_mean": -48.412353515625, + "KL/mean": -77.40523529052734, + "KL/rejected_KL_mean": -106.39811706542969, + "KL/std": 66.31096649169922, + "epoch": 0.3581151832460733, + "fcm_dpo/beta": 0.006731792353093624, + "fcm_dpo/delta": -0.03171330690383911, + "fcm_dpo/margin": 57.98576354980469, + "fcm_dpo/q_t": 0.40904250741004944, + "grad_norm": 76.98529052734375, + "learning_rate": 4.0668899744407567e-07, + "logits/chosen": -0.8132805824279785, + "logits/rejected": -0.8281208872795105, + "logps/chosen": -307.3013916015625, + "logps/ref_chosen": -258.8890380859375, + "logps/ref_rejected": -262.19140625, + "logps/rejected": -368.5894775390625, + "loss": 4.3738, + "margin_dpo/margin_mean": 57.98576354980469, + "margin_dpo/margin_std": 71.38015747070312, + "step": 171 + }, + { + "KL/chosen_KL_mean": -54.201725006103516, + "KL/mean": -70.12786865234375, + "KL/rejected_KL_mean": -86.05400085449219, + "KL/std": 60.457340240478516, + "epoch": 0.36020942408376966, + "fcm_dpo/beta": 0.006770864129066467, + "fcm_dpo/delta": 0.021626032888889313, + "fcm_dpo/margin": 31.85228157043457, + "fcm_dpo/q_t": 0.4491257965564728, + "grad_norm": 90.36054229736328, + "learning_rate": 4.0525824823390043e-07, + "logits/chosen": -0.8342244625091553, + "logits/rejected": -0.850119411945343, + "logps/chosen": -393.22406005859375, + "logps/ref_chosen": -339.0223388671875, + "logps/ref_rejected": -295.78759765625, + "logps/rejected": -381.84161376953125, + "loss": 4.9797, + "margin_dpo/margin_mean": 31.852283477783203, + "margin_dpo/margin_std": 74.22462463378906, + "step": 172 + }, + { + "KL/chosen_KL_mean": -56.15380096435547, + "KL/mean": -77.50936889648438, + "KL/rejected_KL_mean": -98.86494445800781, + "KL/std": 64.10746765136719, + "epoch": 0.362303664921466, + "fcm_dpo/beta": 0.006724307779222727, + "fcm_dpo/delta": -0.025682823732495308, + "fcm_dpo/margin": 42.711151123046875, + "fcm_dpo/q_t": 0.43435436487197876, + "grad_norm": 104.70634460449219, + "learning_rate": 4.0381917299505686e-07, + "logits/chosen": -0.8539585471153259, + "logits/rejected": -0.8501390814781189, + "logps/chosen": -356.2652282714844, + "logps/ref_chosen": -300.1114501953125, + "logps/ref_rejected": -273.78460693359375, + "logps/rejected": -372.6495666503906, + "loss": 4.7538, + "margin_dpo/margin_mean": 42.711151123046875, + "margin_dpo/margin_std": 76.24950408935547, + "step": 173 + }, + { + "KL/chosen_KL_mean": -56.64954376220703, + "KL/mean": -83.61141967773438, + "KL/rejected_KL_mean": -110.57327270507812, + "KL/std": 65.21598815917969, + "epoch": 0.3643979057591623, + "fcm_dpo/beta": 0.00670324731618166, + "fcm_dpo/delta": 0.006629655137658119, + "fcm_dpo/margin": 53.923744201660156, + "fcm_dpo/q_t": 0.41451215744018555, + "grad_norm": 70.95905303955078, + "learning_rate": 4.0237184890078243e-07, + "logits/chosen": -0.8421565294265747, + "logits/rejected": -0.8234596848487854, + "logps/chosen": -391.703369140625, + "logps/ref_chosen": -335.0538635253906, + "logps/ref_rejected": -257.4646911621094, + "logps/rejected": -368.0379638671875, + "loss": 4.4474, + "margin_dpo/margin_mean": 53.923744201660156, + "margin_dpo/margin_std": 69.16473388671875, + "step": 174 + }, + { + "KL/chosen_KL_mean": -73.34457397460938, + "KL/mean": -97.99015808105469, + "KL/rejected_KL_mean": -122.63572692871094, + "KL/std": 74.52360534667969, + "epoch": 0.36649214659685864, + "fcm_dpo/beta": 0.006607607007026672, + "fcm_dpo/delta": -0.04977422207593918, + "fcm_dpo/margin": 49.29114532470703, + "fcm_dpo/q_t": 0.42731624841690063, + "grad_norm": 61.65116882324219, + "learning_rate": 4.00916353566676e-07, + "logits/chosen": -0.8486623764038086, + "logits/rejected": -0.8486083149909973, + "logps/chosen": -357.7401428222656, + "logps/ref_chosen": -284.39556884765625, + "logps/ref_rejected": -283.3876647949219, + "logps/rejected": -406.02337646484375, + "loss": 4.6827, + "margin_dpo/margin_mean": 49.29115295410156, + "margin_dpo/margin_std": 83.50072479248047, + "step": 175 + }, + { + "KL/chosen_KL_mean": -77.07302856445312, + "KL/mean": -96.13621520996094, + "KL/rejected_KL_mean": -115.19940185546875, + "KL/std": 70.39031219482422, + "epoch": 0.36858638743455496, + "fcm_dpo/beta": 0.006414460018277168, + "fcm_dpo/delta": -0.01322941668331623, + "fcm_dpo/margin": 38.12636184692383, + "fcm_dpo/q_t": 0.44413357973098755, + "grad_norm": 84.40768432617188, + "learning_rate": 3.994527650465352e-07, + "logits/chosen": -0.8279335498809814, + "logits/rejected": -0.8326810002326965, + "logps/chosen": -328.8858337402344, + "logps/ref_chosen": -251.81280517578125, + "logps/ref_rejected": -242.05328369140625, + "logps/rejected": -357.2526550292969, + "loss": 4.9587, + "margin_dpo/margin_mean": 38.12635803222656, + "margin_dpo/margin_std": 88.72412872314453, + "step": 176 + }, + { + "KL/chosen_KL_mean": -91.10723876953125, + "KL/mean": -110.1352310180664, + "KL/rejected_KL_mean": -129.1632080078125, + "KL/std": 69.8150634765625, + "epoch": 0.3706806282722513, + "fcm_dpo/beta": 0.006443779915571213, + "fcm_dpo/delta": 0.014410565607249737, + "fcm_dpo/margin": 38.05598449707031, + "fcm_dpo/q_t": 0.4435146152973175, + "grad_norm": 86.33255004882812, + "learning_rate": 3.979811618281705e-07, + "logits/chosen": -0.9177388548851013, + "logits/rejected": -0.8854184746742249, + "logps/chosen": -389.7536315917969, + "logps/ref_chosen": -298.6463928222656, + "logps/ref_rejected": -295.66534423828125, + "logps/rejected": -424.8285217285156, + "loss": 4.9676, + "margin_dpo/margin_mean": 38.05598068237305, + "margin_dpo/margin_std": 86.24855041503906, + "step": 177 + }, + { + "KL/chosen_KL_mean": -78.96491241455078, + "KL/mean": -106.84060668945312, + "KL/rejected_KL_mean": -134.71629333496094, + "KL/std": 75.7911376953125, + "epoch": 0.37277486910994767, + "fcm_dpo/beta": 0.006407511420547962, + "fcm_dpo/delta": -0.014248996041715145, + "fcm_dpo/margin": 55.751380920410156, + "fcm_dpo/q_t": 0.4190220236778259, + "grad_norm": 78.22713470458984, + "learning_rate": 3.9650162282919654e-07, + "logits/chosen": -0.8397620320320129, + "logits/rejected": -0.8298240303993225, + "logps/chosen": -365.22259521484375, + "logps/ref_chosen": -286.2576599121094, + "logps/ref_rejected": -243.97491455078125, + "logps/rejected": -378.69122314453125, + "loss": 4.5792, + "margin_dpo/margin_mean": 55.751380920410156, + "margin_dpo/margin_std": 90.83074951171875, + "step": 178 + }, + { + "KL/chosen_KL_mean": -84.91183471679688, + "KL/mean": -111.3154067993164, + "KL/rejected_KL_mean": -137.71897888183594, + "KL/std": 76.51798248291016, + "epoch": 0.374869109947644, + "fcm_dpo/beta": 0.006282145623117685, + "fcm_dpo/delta": -0.02392883226275444, + "fcm_dpo/margin": 52.80714797973633, + "fcm_dpo/q_t": 0.42275673151016235, + "grad_norm": 82.36101531982422, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": -0.8264894485473633, + "logits/rejected": -0.825171709060669, + "logps/chosen": -344.64892578125, + "logps/ref_chosen": -259.737060546875, + "logps/ref_rejected": -277.8813171386719, + "logps/rejected": -415.6003112792969, + "loss": 4.6261, + "margin_dpo/margin_mean": 52.80714797973633, + "margin_dpo/margin_std": 83.93762969970703, + "step": 179 + }, + { + "KL/chosen_KL_mean": -87.58425903320312, + "KL/mean": -113.29141235351562, + "KL/rejected_KL_mean": -138.99856567382812, + "KL/std": 72.59463500976562, + "epoch": 0.3769633507853403, + "fcm_dpo/beta": 0.006279164459556341, + "fcm_dpo/delta": 0.013550683856010437, + "fcm_dpo/margin": 51.414302825927734, + "fcm_dpo/q_t": 0.4277075529098511, + "grad_norm": 102.1092758178711, + "learning_rate": 3.935190552834828e-07, + "logits/chosen": -0.8694858551025391, + "logits/rejected": -0.8932338356971741, + "logps/chosen": -354.8931579589844, + "logps/ref_chosen": -267.30889892578125, + "logps/ref_rejected": -230.4376983642578, + "logps/rejected": -369.436279296875, + "loss": 4.6452, + "margin_dpo/margin_mean": 51.41429901123047, + "margin_dpo/margin_std": 85.73615264892578, + "step": 180 + }, + { + "KL/chosen_KL_mean": -95.89186096191406, + "KL/mean": -118.81684112548828, + "KL/rejected_KL_mean": -141.7418212890625, + "KL/std": 74.99784088134766, + "epoch": 0.37905759162303665, + "fcm_dpo/beta": 0.006303788162767887, + "fcm_dpo/delta": 0.002721858210861683, + "fcm_dpo/margin": 45.849979400634766, + "fcm_dpo/q_t": 0.43346357345581055, + "grad_norm": 80.63015747070312, + "learning_rate": 3.920161866827889e-07, + "logits/chosen": -0.857856810092926, + "logits/rejected": -0.856174886226654, + "logps/chosen": -396.38323974609375, + "logps/ref_chosen": -300.49139404296875, + "logps/ref_rejected": -278.98284912109375, + "logps/rejected": -420.72467041015625, + "loss": 4.7731, + "margin_dpo/margin_mean": 45.849979400634766, + "margin_dpo/margin_std": 86.88410186767578, + "step": 181 + }, + { + "KL/chosen_KL_mean": -90.28527069091797, + "KL/mean": -123.19674682617188, + "KL/rejected_KL_mean": -156.1082000732422, + "KL/std": 74.51239013671875, + "epoch": 0.381151832460733, + "fcm_dpo/beta": 0.00616471329703927, + "fcm_dpo/delta": -0.06726302951574326, + "fcm_dpo/margin": 65.82292938232422, + "fcm_dpo/q_t": 0.40653538703918457, + "grad_norm": 70.58544158935547, + "learning_rate": 3.90505702185e-07, + "logits/chosen": -0.8527212142944336, + "logits/rejected": -0.8740633726119995, + "logps/chosen": -369.7834167480469, + "logps/ref_chosen": -279.4981689453125, + "logps/ref_rejected": -263.6926574707031, + "logps/rejected": -419.8008117675781, + "loss": 4.3893, + "margin_dpo/margin_mean": 65.82292938232422, + "margin_dpo/margin_std": 87.4244613647461, + "step": 182 + }, + { + "KL/chosen_KL_mean": -88.05329895019531, + "KL/mean": -115.47635650634766, + "KL/rejected_KL_mean": -142.8994140625, + "KL/std": 75.31936645507812, + "epoch": 0.3832460732984293, + "fcm_dpo/beta": 0.006069937255233526, + "fcm_dpo/delta": 0.017182346433401108, + "fcm_dpo/margin": 54.846107482910156, + "fcm_dpo/q_t": 0.42420393228530884, + "grad_norm": 76.18987274169922, + "learning_rate": 3.889876827928156e-07, + "logits/chosen": -0.8800171613693237, + "logits/rejected": -0.8790852427482605, + "logps/chosen": -359.259033203125, + "logps/ref_chosen": -271.2057189941406, + "logps/ref_rejected": -243.91549682617188, + "logps/rejected": -386.8149108886719, + "loss": 4.6149, + "margin_dpo/margin_mean": 54.846107482910156, + "margin_dpo/margin_std": 89.88021087646484, + "step": 183 + }, + { + "KL/chosen_KL_mean": -83.444580078125, + "KL/mean": -122.65955352783203, + "KL/rejected_KL_mean": -161.87454223632812, + "KL/std": 84.31695556640625, + "epoch": 0.38534031413612563, + "fcm_dpo/beta": 0.00580790126696229, + "fcm_dpo/delta": -0.07667214423418045, + "fcm_dpo/margin": 78.4299545288086, + "fcm_dpo/q_t": 0.39444056153297424, + "grad_norm": 72.20956420898438, + "learning_rate": 3.874622099130087e-07, + "logits/chosen": -0.929442822933197, + "logits/rejected": -0.9039252996444702, + "logps/chosen": -401.89031982421875, + "logps/ref_chosen": -318.4457702636719, + "logps/ref_rejected": -266.640869140625, + "logps/rejected": -428.5154113769531, + "loss": 4.2258, + "margin_dpo/margin_mean": 78.4299545288086, + "margin_dpo/margin_std": 94.27364349365234, + "step": 184 + }, + { + "KL/chosen_KL_mean": -90.38389587402344, + "KL/mean": -119.19790649414062, + "KL/rejected_KL_mean": -148.01194763183594, + "KL/std": 88.37677764892578, + "epoch": 0.387434554973822, + "fcm_dpo/beta": 0.005671571008861065, + "fcm_dpo/delta": -0.00434712041169405, + "fcm_dpo/margin": 57.628047943115234, + "fcm_dpo/q_t": 0.42499980330467224, + "grad_norm": 110.43806457519531, + "learning_rate": 3.859293653520604e-07, + "logits/chosen": -0.8981891870498657, + "logits/rejected": -0.8912757635116577, + "logps/chosen": -364.69268798828125, + "logps/ref_chosen": -274.308837890625, + "logps/ref_rejected": -260.7274169921875, + "logps/rejected": -408.7393493652344, + "loss": 4.63, + "margin_dpo/margin_mean": 57.62804412841797, + "margin_dpo/margin_std": 93.35250091552734, + "step": 185 + }, + { + "KL/chosen_KL_mean": -89.22308349609375, + "KL/mean": -115.18913269042969, + "KL/rejected_KL_mean": -141.15518188476562, + "KL/std": 79.0693359375, + "epoch": 0.38952879581151834, + "fcm_dpo/beta": 0.005763459485024214, + "fcm_dpo/delta": 0.013718879781663418, + "fcm_dpo/margin": 51.932098388671875, + "fcm_dpo/q_t": 0.4305761456489563, + "grad_norm": 92.03166961669922, + "learning_rate": 3.8438923131177237e-07, + "logits/chosen": -0.9018892049789429, + "logits/rejected": -0.9007343649864197, + "logps/chosen": -388.22845458984375, + "logps/ref_chosen": -299.00537109375, + "logps/ref_rejected": -274.4014587402344, + "logps/rejected": -415.5566101074219, + "loss": 4.6983, + "margin_dpo/margin_mean": 51.932098388671875, + "margin_dpo/margin_std": 86.81627655029297, + "step": 186 + }, + { + "KL/chosen_KL_mean": -96.13522338867188, + "KL/mean": -119.4954605102539, + "KL/rejected_KL_mean": -142.85569763183594, + "KL/std": 82.50918579101562, + "epoch": 0.39162303664921466, + "fcm_dpo/beta": 0.0057730223052203655, + "fcm_dpo/delta": 0.004285093396902084, + "fcm_dpo/margin": 46.72047805786133, + "fcm_dpo/q_t": 0.43553128838539124, + "grad_norm": 85.7626953125, + "learning_rate": 3.828418903848593e-07, + "logits/chosen": -0.8542200922966003, + "logits/rejected": -0.8379209637641907, + "logps/chosen": -425.96051025390625, + "logps/ref_chosen": -329.8253173828125, + "logps/ref_rejected": -263.73175048828125, + "logps/rejected": -406.58746337890625, + "loss": 4.8707, + "margin_dpo/margin_mean": 46.72047805786133, + "margin_dpo/margin_std": 97.68164825439453, + "step": 187 + }, + { + "KL/chosen_KL_mean": -85.97469329833984, + "KL/mean": -112.66793823242188, + "KL/rejected_KL_mean": -139.36119079589844, + "KL/std": 84.3029556274414, + "epoch": 0.393717277486911, + "fcm_dpo/beta": 0.0057581509463489056, + "fcm_dpo/delta": -0.01138460636138916, + "fcm_dpo/margin": 53.38649368286133, + "fcm_dpo/q_t": 0.43061012029647827, + "grad_norm": 67.76911926269531, + "learning_rate": 3.812874255505191e-07, + "logits/chosen": -0.8586629629135132, + "logits/rejected": -0.8478258848190308, + "logps/chosen": -348.9803161621094, + "logps/ref_chosen": -263.005615234375, + "logps/ref_rejected": -247.08668518066406, + "logps/rejected": -386.4478454589844, + "loss": 4.7691, + "margin_dpo/margin_mean": 53.38649368286133, + "margin_dpo/margin_std": 103.5580825805664, + "step": 188 + }, + { + "KL/chosen_KL_mean": -68.06344604492188, + "KL/mean": -101.9673080444336, + "KL/rejected_KL_mean": -135.8711700439453, + "KL/std": 82.10550689697266, + "epoch": 0.3958115183246073, + "fcm_dpo/beta": 0.00560600683093071, + "fcm_dpo/delta": -0.03640016168355942, + "fcm_dpo/margin": 67.8077392578125, + "fcm_dpo/q_t": 0.4118382930755615, + "grad_norm": 69.77622985839844, + "learning_rate": 3.797259201699833e-07, + "logits/chosen": -0.880431592464447, + "logits/rejected": -0.8870152235031128, + "logps/chosen": -341.0238342285156, + "logps/ref_chosen": -272.96038818359375, + "logps/ref_rejected": -275.13238525390625, + "logps/rejected": -411.0035705566406, + "loss": 4.4049, + "margin_dpo/margin_mean": 67.80773162841797, + "margin_dpo/margin_std": 85.12550354003906, + "step": 189 + }, + { + "KL/chosen_KL_mean": -74.26818084716797, + "KL/mean": -101.46267700195312, + "KL/rejected_KL_mean": -128.65719604492188, + "KL/std": 77.80656433105469, + "epoch": 0.39790575916230364, + "fcm_dpo/beta": 0.005546726752072573, + "fcm_dpo/delta": -0.009571824222803116, + "fcm_dpo/margin": 54.38901138305664, + "fcm_dpo/q_t": 0.4297546446323395, + "grad_norm": 93.16829681396484, + "learning_rate": 3.781574579820464e-07, + "logits/chosen": -0.8819738030433655, + "logits/rejected": -0.841058611869812, + "logps/chosen": -332.06573486328125, + "logps/ref_chosen": -257.79754638671875, + "logps/ref_rejected": -225.2164306640625, + "logps/rejected": -353.8736572265625, + "loss": 4.657, + "margin_dpo/margin_mean": 54.38901138305664, + "margin_dpo/margin_std": 85.22750091552734, + "step": 190 + }, + { + "KL/chosen_KL_mean": -66.46964263916016, + "KL/mean": -94.91885375976562, + "KL/rejected_KL_mean": -123.3680648803711, + "KL/std": 79.13052368164062, + "epoch": 0.4, + "fcm_dpo/beta": 0.0055416785180568695, + "fcm_dpo/delta": 0.0067793321795761585, + "fcm_dpo/margin": 56.89841842651367, + "fcm_dpo/q_t": 0.4267407953739166, + "grad_norm": 61.709129333496094, + "learning_rate": 3.765821230985757e-07, + "logits/chosen": -0.9132006764411926, + "logits/rejected": -0.9146799445152283, + "logps/chosen": -310.3281555175781, + "logps/ref_chosen": -243.8585205078125, + "logps/ref_rejected": -245.12136840820312, + "logps/rejected": -368.48944091796875, + "loss": 4.6375, + "margin_dpo/margin_mean": 56.89841842651367, + "margin_dpo/margin_std": 91.43920135498047, + "step": 191 + }, + { + "KL/chosen_KL_mean": -79.08763122558594, + "KL/mean": -101.60113525390625, + "KL/rejected_KL_mean": -124.11463928222656, + "KL/std": 76.82417297363281, + "epoch": 0.40209424083769635, + "fcm_dpo/beta": 0.0056621660478413105, + "fcm_dpo/delta": 0.029165830463171005, + "fcm_dpo/margin": 45.02701187133789, + "fcm_dpo/q_t": 0.44025444984436035, + "grad_norm": 73.44487762451172, + "learning_rate": 3.75e-07, + "logits/chosen": -0.8472084999084473, + "logits/rejected": -0.8306607604026794, + "logps/chosen": -346.06756591796875, + "logps/ref_chosen": -266.9799499511719, + "logps/ref_rejected": -260.1697082519531, + "logps/rejected": -384.2843017578125, + "loss": 4.824, + "margin_dpo/margin_mean": 45.02701187133789, + "margin_dpo/margin_std": 83.66022491455078, + "step": 192 + }, + { + "KL/chosen_KL_mean": -94.40106964111328, + "KL/mean": -121.47966003417969, + "KL/rejected_KL_mean": -148.55825805664062, + "KL/std": 83.79367065429688, + "epoch": 0.4041884816753927, + "fcm_dpo/beta": 0.005736154038459063, + "fcm_dpo/delta": 0.005536096170544624, + "fcm_dpo/margin": 54.157196044921875, + "fcm_dpo/q_t": 0.42890992760658264, + "grad_norm": 72.95458221435547, + "learning_rate": 3.734111735307796e-07, + "logits/chosen": -0.8859913945198059, + "logits/rejected": -0.8564121723175049, + "logps/chosen": -374.654296875, + "logps/ref_chosen": -280.25323486328125, + "logps/ref_rejected": -291.0348815917969, + "logps/rejected": -439.5931396484375, + "loss": 4.7054, + "margin_dpo/margin_mean": 54.157196044921875, + "margin_dpo/margin_std": 94.62388610839844, + "step": 193 + }, + { + "KL/chosen_KL_mean": -92.71996307373047, + "KL/mean": -116.64244842529297, + "KL/rejected_KL_mean": -140.56492614746094, + "KL/std": 82.40608978271484, + "epoch": 0.406282722513089, + "fcm_dpo/beta": 0.00563084939494729, + "fcm_dpo/delta": -0.02582578919827938, + "fcm_dpo/margin": 47.844966888427734, + "fcm_dpo/q_t": 0.43761441111564636, + "grad_norm": 99.4243392944336, + "learning_rate": 3.7181572889485623e-07, + "logits/chosen": -0.8625931739807129, + "logits/rejected": -0.8507343530654907, + "logps/chosen": -380.8594055175781, + "logps/ref_chosen": -288.13946533203125, + "logps/ref_rejected": -251.31529235839844, + "logps/rejected": -391.8802185058594, + "loss": 4.7859, + "margin_dpo/margin_mean": 47.844966888427734, + "margin_dpo/margin_std": 85.31089782714844, + "step": 194 + }, + { + "KL/chosen_KL_mean": -97.91946411132812, + "KL/mean": -119.41505432128906, + "KL/rejected_KL_mean": -140.91062927246094, + "KL/std": 80.02857971191406, + "epoch": 0.4083769633507853, + "fcm_dpo/beta": 0.005613365676254034, + "fcm_dpo/delta": -0.0017420882359147072, + "fcm_dpo/margin": 42.99113464355469, + "fcm_dpo/q_t": 0.44507643580436707, + "grad_norm": 73.5301513671875, + "learning_rate": 3.7021375165108377e-07, + "logits/chosen": -0.8627254366874695, + "logits/rejected": -0.8665878176689148, + "logps/chosen": -371.92010498046875, + "logps/ref_chosen": -274.0006408691406, + "logps/ref_rejected": -280.22723388671875, + "logps/rejected": -421.13787841796875, + "loss": 4.8972, + "margin_dpo/margin_mean": 42.99113464355469, + "margin_dpo/margin_std": 88.47681427001953, + "step": 195 + }, + { + "KL/chosen_KL_mean": -82.35733032226562, + "KL/mean": -109.65511322021484, + "KL/rejected_KL_mean": -136.95291137695312, + "KL/std": 82.34046936035156, + "epoch": 0.41047120418848165, + "fcm_dpo/beta": 0.005610158666968346, + "fcm_dpo/delta": -0.0006429278291761875, + "fcm_dpo/margin": 54.595584869384766, + "fcm_dpo/q_t": 0.42899465560913086, + "grad_norm": 76.89717864990234, + "learning_rate": 3.6860532770864005e-07, + "logits/chosen": -0.8541459441184998, + "logits/rejected": -0.8584092855453491, + "logps/chosen": -357.25799560546875, + "logps/ref_chosen": -274.90069580078125, + "logps/ref_rejected": -248.7281951904297, + "logps/rejected": -385.6811218261719, + "loss": 4.6883, + "margin_dpo/margin_mean": 54.595584869384766, + "margin_dpo/margin_std": 92.98194122314453, + "step": 196 + }, + { + "KL/chosen_KL_mean": -92.13257598876953, + "KL/mean": -123.7594985961914, + "KL/rejected_KL_mean": -155.38644409179688, + "KL/std": 82.77664184570312, + "epoch": 0.41256544502617803, + "fcm_dpo/beta": 0.00550233107060194, + "fcm_dpo/delta": -0.03711705654859543, + "fcm_dpo/margin": 63.25384521484375, + "fcm_dpo/q_t": 0.4192579984664917, + "grad_norm": 90.26445770263672, + "learning_rate": 3.6699054332241985e-07, + "logits/chosen": -0.8871943950653076, + "logits/rejected": -0.8706080913543701, + "logps/chosen": -401.66741943359375, + "logps/ref_chosen": -309.5348205566406, + "logps/ref_rejected": -264.3179931640625, + "logps/rejected": -419.7044372558594, + "loss": 4.54, + "margin_dpo/margin_mean": 63.25384521484375, + "margin_dpo/margin_std": 89.81802368164062, + "step": 197 + }, + { + "KL/chosen_KL_mean": -87.10678100585938, + "KL/mean": -119.78683471679688, + "KL/rejected_KL_mean": -152.46690368652344, + "KL/std": 88.0517807006836, + "epoch": 0.41465968586387436, + "fcm_dpo/beta": 0.005422430112957954, + "fcm_dpo/delta": 0.002035717014223337, + "fcm_dpo/margin": 65.360107421875, + "fcm_dpo/q_t": 0.41849538683891296, + "grad_norm": 61.8067512512207, + "learning_rate": 3.653694850884091e-07, + "logits/chosen": -0.8917360305786133, + "logits/rejected": -0.8658944964408875, + "logps/chosen": -388.1202392578125, + "logps/ref_chosen": -301.0134582519531, + "logps/ref_rejected": -292.84185791015625, + "logps/rejected": -445.3087158203125, + "loss": 4.5293, + "margin_dpo/margin_mean": 65.36009979248047, + "margin_dpo/margin_std": 97.01239013671875, + "step": 198 + }, + { + "KL/chosen_KL_mean": -97.43031311035156, + "KL/mean": -124.94877624511719, + "KL/rejected_KL_mean": -152.46722412109375, + "KL/std": 83.84455871582031, + "epoch": 0.4167539267015707, + "fcm_dpo/beta": 0.0055153826251626015, + "fcm_dpo/delta": 0.018592506647109985, + "fcm_dpo/margin": 55.036903381347656, + "fcm_dpo/q_t": 0.4302946627140045, + "grad_norm": 96.4948501586914, + "learning_rate": 3.6374223993904124e-07, + "logits/chosen": -0.8654062747955322, + "logits/rejected": -0.8202941417694092, + "logps/chosen": -362.03619384765625, + "logps/ref_chosen": -264.6058654785156, + "logps/ref_rejected": -214.9014892578125, + "logps/rejected": -367.36871337890625, + "loss": 4.6993, + "margin_dpo/margin_mean": 55.03690719604492, + "margin_dpo/margin_std": 95.58619689941406, + "step": 199 + }, + { + "KL/chosen_KL_mean": -107.61213684082031, + "KL/mean": -131.58660888671875, + "KL/rejected_KL_mean": -155.56106567382812, + "KL/std": 85.42179870605469, + "epoch": 0.418848167539267, + "fcm_dpo/beta": 0.005476394202560186, + "fcm_dpo/delta": -0.020585589110851288, + "fcm_dpo/margin": 47.948944091796875, + "fcm_dpo/q_t": 0.4395952820777893, + "grad_norm": 106.97425842285156, + "learning_rate": 3.621088951385353e-07, + "logits/chosen": -0.9183908700942993, + "logits/rejected": -0.8932676315307617, + "logps/chosen": -431.7709655761719, + "logps/ref_chosen": -324.1588134765625, + "logps/ref_rejected": -277.80218505859375, + "logps/rejected": -433.36328125, + "loss": 4.9173, + "margin_dpo/margin_mean": 47.948944091796875, + "margin_dpo/margin_std": 107.63665008544922, + "step": 200 + }, + { + "epoch": 0.418848167539267, + "eval_KL/chosen_KL_mean": -118.4775161743164, + "eval_KL/mean": -146.10684204101562, + "eval_KL/rejected_KL_mean": -173.73617553710938, + "eval_KL/std": 87.68894958496094, + "eval_fcm_dpo/beta": 0.005431666970252991, + "eval_logits/chosen": -0.9033797979354858, + "eval_logits/rejected": -0.8850231170654297, + "eval_logps/chosen": -406.3043212890625, + "eval_logps/ref_chosen": -287.8268127441406, + "eval_logps/ref_rejected": -266.9300231933594, + "eval_logps/rejected": -440.666259765625, + "eval_loss": 0.5937886834144592, + "eval_margin_dpo/margin_mean": 55.258663177490234, + "eval_margin_dpo/margin_std": 96.29785919189453, + "eval_runtime": 78.3994, + "eval_samples_per_second": 25.51, + "eval_steps_per_second": 3.189, + "step": 200 + }, + { + "KL/chosen_KL_mean": -114.04182434082031, + "KL/mean": -142.33677673339844, + "KL/rejected_KL_mean": -170.63168334960938, + "KL/std": 90.65510559082031, + "epoch": 0.42094240837696334, + "fcm_dpo/beta": 0.005450923927128315, + "fcm_dpo/delta": -0.003941504284739494, + "fcm_dpo/margin": 56.58985900878906, + "fcm_dpo/q_t": 0.4291457533836365, + "grad_norm": 69.15496826171875, + "learning_rate": 3.604695382782159e-07, + "logits/chosen": -0.8902816772460938, + "logits/rejected": -0.8751832246780396, + "logps/chosen": -385.53753662109375, + "logps/ref_chosen": -271.49566650390625, + "logps/ref_rejected": -245.71414184570312, + "logps/rejected": -416.3458251953125, + "loss": 4.7244, + "margin_dpo/margin_mean": 56.58985900878906, + "margin_dpo/margin_std": 102.68513488769531, + "step": 201 + }, + { + "KL/chosen_KL_mean": -119.89775848388672, + "KL/mean": -146.6784210205078, + "KL/rejected_KL_mean": -173.45907592773438, + "KL/std": 88.81134796142578, + "epoch": 0.42303664921465967, + "fcm_dpo/beta": 0.005403751973062754, + "fcm_dpo/delta": 9.850505739450455e-05, + "fcm_dpo/margin": 53.561317443847656, + "fcm_dpo/q_t": 0.43379461765289307, + "grad_norm": 89.58255767822266, + "learning_rate": 3.588242572718162e-07, + "logits/chosen": -0.9021543264389038, + "logits/rejected": -0.8883557915687561, + "logps/chosen": -391.9956970214844, + "logps/ref_chosen": -272.0979309082031, + "logps/ref_rejected": -235.94805908203125, + "logps/rejected": -409.4071350097656, + "loss": 4.7836, + "margin_dpo/margin_mean": 53.561317443847656, + "margin_dpo/margin_std": 102.62480163574219, + "step": 202 + }, + { + "KL/chosen_KL_mean": -130.4102783203125, + "KL/mean": -151.12998962402344, + "KL/rejected_KL_mean": -171.84970092773438, + "KL/std": 87.56797790527344, + "epoch": 0.42513089005235605, + "fcm_dpo/beta": 0.00546378456056118, + "fcm_dpo/delta": 0.014473012648522854, + "fcm_dpo/margin": 41.439414978027344, + "fcm_dpo/q_t": 0.44730111956596375, + "grad_norm": 102.8392562866211, + "learning_rate": 3.571731403507635e-07, + "logits/chosen": -0.8795283436775208, + "logits/rejected": -0.8848550319671631, + "logps/chosen": -410.6324462890625, + "logps/ref_chosen": -280.2221374511719, + "logps/ref_rejected": -251.79798889160156, + "logps/rejected": -423.6476745605469, + "loss": 4.9634, + "margin_dpo/margin_mean": 41.439414978027344, + "margin_dpo/margin_std": 95.13355255126953, + "step": 203 + }, + { + "KL/chosen_KL_mean": -132.2213592529297, + "KL/mean": -160.99591064453125, + "KL/rejected_KL_mean": -189.77044677734375, + "KL/std": 87.84689331054688, + "epoch": 0.4272251308900524, + "fcm_dpo/beta": 0.005502759013324976, + "fcm_dpo/delta": 0.007985341362655163, + "fcm_dpo/margin": 57.549095153808594, + "fcm_dpo/q_t": 0.42689085006713867, + "grad_norm": 125.6895751953125, + "learning_rate": 3.5551627605944746e-07, + "logits/chosen": -0.9232524633407593, + "logits/rejected": -0.8936776518821716, + "logps/chosen": -451.0173645019531, + "logps/ref_chosen": -318.7960510253906, + "logps/ref_rejected": -269.69921875, + "logps/rejected": -459.4697265625, + "loss": 4.671, + "margin_dpo/margin_mean": 57.549102783203125, + "margin_dpo/margin_std": 98.45879364013672, + "step": 204 + }, + { + "KL/chosen_KL_mean": -136.3469696044922, + "KL/mean": -168.44393920898438, + "KL/rejected_KL_mean": -200.54092407226562, + "KL/std": 92.91146850585938, + "epoch": 0.4293193717277487, + "fcm_dpo/beta": 0.005452723708003759, + "fcm_dpo/delta": -0.045753512531518936, + "fcm_dpo/margin": 64.19393920898438, + "fcm_dpo/q_t": 0.421352356672287, + "grad_norm": 128.93753051757812, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": -0.8728401064872742, + "logits/rejected": -0.841010570526123, + "logps/chosen": -420.10906982421875, + "logps/ref_chosen": -283.7620544433594, + "logps/ref_rejected": -297.69439697265625, + "logps/rejected": -498.2353210449219, + "loss": 4.6025, + "margin_dpo/margin_mean": 64.19393920898438, + "margin_dpo/margin_std": 103.74689483642578, + "step": 205 + }, + { + "KL/chosen_KL_mean": -142.62896728515625, + "KL/mean": -167.98255920410156, + "KL/rejected_KL_mean": -193.33612060546875, + "KL/std": 92.03362274169922, + "epoch": 0.431413612565445, + "fcm_dpo/beta": 0.005307645071297884, + "fcm_dpo/delta": -0.0065862322226166725, + "fcm_dpo/margin": 50.70718002319336, + "fcm_dpo/q_t": 0.43794897198677063, + "grad_norm": 107.9933090209961, + "learning_rate": 3.5218566107988867e-07, + "logits/chosen": -0.895841121673584, + "logits/rejected": -0.9164851903915405, + "logps/chosen": -436.2928466796875, + "logps/ref_chosen": -293.66387939453125, + "logps/ref_rejected": -291.3056640625, + "logps/rejected": -484.6418151855469, + "loss": 4.8515, + "margin_dpo/margin_mean": 50.70718002319336, + "margin_dpo/margin_std": 103.65234375, + "step": 206 + }, + { + "KL/chosen_KL_mean": -129.6578826904297, + "KL/mean": -155.3979034423828, + "KL/rejected_KL_mean": -181.13790893554688, + "KL/std": 103.08729553222656, + "epoch": 0.43350785340314135, + "fcm_dpo/beta": 0.005296694580465555, + "fcm_dpo/delta": 0.01233250554651022, + "fcm_dpo/margin": 51.480037689208984, + "fcm_dpo/q_t": 0.43691202998161316, + "grad_norm": 71.85640716552734, + "learning_rate": 3.505120890024195e-07, + "logits/chosen": -0.8434122800827026, + "logits/rejected": -0.8525888323783875, + "logps/chosen": -400.1929931640625, + "logps/ref_chosen": -270.5350646972656, + "logps/ref_rejected": -278.7747497558594, + "logps/rejected": -459.91265869140625, + "loss": 4.9192, + "margin_dpo/margin_mean": 51.480037689208984, + "margin_dpo/margin_std": 117.58082580566406, + "step": 207 + }, + { + "KL/chosen_KL_mean": -137.4970703125, + "KL/mean": -168.57623291015625, + "KL/rejected_KL_mean": -199.65541076660156, + "KL/std": 101.80587005615234, + "epoch": 0.4356020942408377, + "fcm_dpo/beta": 0.0053030322305858135, + "fcm_dpo/delta": -0.015698019415140152, + "fcm_dpo/margin": 62.15835952758789, + "fcm_dpo/q_t": 0.42619529366493225, + "grad_norm": 76.81256103515625, + "learning_rate": 3.4883312676665534e-07, + "logits/chosen": -0.9078364968299866, + "logits/rejected": -0.8600209355354309, + "logps/chosen": -417.0798034667969, + "logps/ref_chosen": -279.582763671875, + "logps/ref_rejected": -290.041015625, + "logps/rejected": -489.6964111328125, + "loss": 4.6342, + "margin_dpo/margin_mean": 62.15835952758789, + "margin_dpo/margin_std": 105.14048767089844, + "step": 208 + }, + { + "KL/chosen_KL_mean": -139.73585510253906, + "KL/mean": -159.71441650390625, + "KL/rejected_KL_mean": -179.6929931640625, + "KL/std": 94.56495666503906, + "epoch": 0.437696335078534, + "fcm_dpo/beta": 0.005294729024171829, + "fcm_dpo/delta": 0.011330154724419117, + "fcm_dpo/margin": 39.9571533203125, + "fcm_dpo/q_t": 0.4502761662006378, + "grad_norm": 107.32901763916016, + "learning_rate": 3.4714886441024573e-07, + "logits/chosen": -0.8233649730682373, + "logits/rejected": -0.8163270950317383, + "logps/chosen": -458.6083679199219, + "logps/ref_chosen": -318.8725280761719, + "logps/ref_rejected": -270.64324951171875, + "logps/rejected": -450.33624267578125, + "loss": 5.0317, + "margin_dpo/margin_mean": 39.9571533203125, + "margin_dpo/margin_std": 99.94036102294922, + "step": 209 + }, + { + "KL/chosen_KL_mean": -127.69154357910156, + "KL/mean": -158.0561981201172, + "KL/rejected_KL_mean": -188.4208526611328, + "KL/std": 90.50584411621094, + "epoch": 0.4397905759162304, + "fcm_dpo/beta": 0.0052696773782372475, + "fcm_dpo/delta": -0.023277752101421356, + "fcm_dpo/margin": 60.729339599609375, + "fcm_dpo/q_t": 0.4259607493877411, + "grad_norm": 83.70458984375, + "learning_rate": 3.454593922550693e-07, + "logits/chosen": -0.8556398153305054, + "logits/rejected": -0.8458552360534668, + "logps/chosen": -410.83184814453125, + "logps/ref_chosen": -283.14031982421875, + "logps/ref_rejected": -287.2986755371094, + "logps/rejected": -475.71954345703125, + "loss": 4.6405, + "margin_dpo/margin_mean": 60.729331970214844, + "margin_dpo/margin_std": 97.90538024902344, + "step": 210 + }, + { + "KL/chosen_KL_mean": -111.53688049316406, + "KL/mean": -149.20333862304688, + "KL/rejected_KL_mean": -186.8698272705078, + "KL/std": 91.30079650878906, + "epoch": 0.4418848167539267, + "fcm_dpo/beta": 0.005068170838057995, + "fcm_dpo/delta": -0.047747448086738586, + "fcm_dpo/margin": 75.33293151855469, + "fcm_dpo/q_t": 0.4104166328907013, + "grad_norm": 83.38289642333984, + "learning_rate": 3.4376480090239047e-07, + "logits/chosen": -0.8847814798355103, + "logits/rejected": -0.8608183860778809, + "logps/chosen": -387.9597473144531, + "logps/ref_chosen": -276.4228515625, + "logps/ref_rejected": -252.40603637695312, + "logps/rejected": -439.2758483886719, + "loss": 4.3908, + "margin_dpo/margin_mean": 75.33293151855469, + "margin_dpo/margin_std": 90.77347564697266, + "step": 211 + }, + { + "KL/chosen_KL_mean": -120.57503509521484, + "KL/mean": -145.82603454589844, + "KL/rejected_KL_mean": -171.07704162597656, + "KL/std": 89.40132141113281, + "epoch": 0.44397905759162304, + "fcm_dpo/beta": 0.005025006365031004, + "fcm_dpo/delta": 0.008119989186525345, + "fcm_dpo/margin": 50.502010345458984, + "fcm_dpo/q_t": 0.4400903582572937, + "grad_norm": 74.55123138427734, + "learning_rate": 3.4206518122800055e-07, + "logits/chosen": -0.8634406328201294, + "logits/rejected": -0.8582726716995239, + "logps/chosen": -392.28057861328125, + "logps/ref_chosen": -271.7055358886719, + "logps/ref_rejected": -241.18511962890625, + "logps/rejected": -412.26220703125, + "loss": 4.8539, + "margin_dpo/margin_mean": 50.50200653076172, + "margin_dpo/margin_std": 98.8302001953125, + "step": 212 + }, + { + "KL/chosen_KL_mean": -118.58316802978516, + "KL/mean": -146.1175537109375, + "KL/rejected_KL_mean": -173.6519317626953, + "KL/std": 103.11312103271484, + "epoch": 0.44607329842931936, + "fcm_dpo/beta": 0.0050498321652412415, + "fcm_dpo/delta": -0.005192122422158718, + "fcm_dpo/margin": 55.06875991821289, + "fcm_dpo/q_t": 0.43775975704193115, + "grad_norm": 82.58657836914062, + "learning_rate": 3.403606243773448e-07, + "logits/chosen": -0.8551130294799805, + "logits/rejected": -0.8685761094093323, + "logps/chosen": -420.88079833984375, + "logps/ref_chosen": -302.2976379394531, + "logps/ref_rejected": -303.6202087402344, + "logps/rejected": -477.2721252441406, + "loss": 4.8435, + "margin_dpo/margin_mean": 55.06875991821289, + "margin_dpo/margin_std": 114.744384765625, + "step": 213 + }, + { + "KL/chosen_KL_mean": -125.1562728881836, + "KL/mean": -158.9164581298828, + "KL/rejected_KL_mean": -192.6766357421875, + "KL/std": 91.4408950805664, + "epoch": 0.4481675392670157, + "fcm_dpo/beta": 0.004997848533093929, + "fcm_dpo/delta": -0.02767416462302208, + "fcm_dpo/margin": 67.52037811279297, + "fcm_dpo/q_t": 0.42145538330078125, + "grad_norm": 90.98759460449219, + "learning_rate": 3.3865122176063385e-07, + "logits/chosen": -0.8687431216239929, + "logits/rejected": -0.8687351942062378, + "logps/chosen": -397.2889099121094, + "logps/ref_chosen": -272.13262939453125, + "logps/ref_rejected": -294.82354736328125, + "logps/rejected": -487.50018310546875, + "loss": 4.535, + "margin_dpo/margin_mean": 67.5203857421875, + "margin_dpo/margin_std": 91.6003189086914, + "step": 214 + }, + { + "KL/chosen_KL_mean": -128.5072479248047, + "KL/mean": -150.73866271972656, + "KL/rejected_KL_mean": -172.97007751464844, + "KL/std": 100.23798370361328, + "epoch": 0.450261780104712, + "fcm_dpo/beta": 0.004917190410196781, + "fcm_dpo/delta": -0.015194178558886051, + "fcm_dpo/margin": 44.46282958984375, + "fcm_dpo/q_t": 0.45000001788139343, + "grad_norm": 86.00048828125, + "learning_rate": 3.3693706504794243e-07, + "logits/chosen": -0.9164157509803772, + "logits/rejected": -0.8946561813354492, + "logps/chosen": -419.8855285644531, + "logps/ref_chosen": -291.3782958984375, + "logps/ref_rejected": -261.05792236328125, + "logps/rejected": -434.02801513671875, + "loss": 4.9817, + "margin_dpo/margin_mean": 44.46282958984375, + "margin_dpo/margin_std": 101.5790786743164, + "step": 215 + }, + { + "KL/chosen_KL_mean": -115.26673889160156, + "KL/mean": -144.14666748046875, + "KL/rejected_KL_mean": -173.02658081054688, + "KL/std": 91.47891998291016, + "epoch": 0.4523560209424084, + "fcm_dpo/beta": 0.004956618882715702, + "fcm_dpo/delta": 0.033862657845020294, + "fcm_dpo/margin": 57.759849548339844, + "fcm_dpo/q_t": 0.4338792860507965, + "grad_norm": 67.37702178955078, + "learning_rate": 3.3521824616429284e-07, + "logits/chosen": -0.9285804033279419, + "logits/rejected": -0.9208376407623291, + "logps/chosen": -453.7721862792969, + "logps/ref_chosen": -338.50543212890625, + "logps/ref_rejected": -305.76104736328125, + "logps/rejected": -478.7876281738281, + "loss": 4.7734, + "margin_dpo/margin_mean": 57.759849548339844, + "margin_dpo/margin_std": 107.90360260009766, + "step": 216 + }, + { + "KL/chosen_KL_mean": -118.00670623779297, + "KL/mean": -152.04202270507812, + "KL/rejected_KL_mean": -186.07737731933594, + "KL/std": 93.6181869506836, + "epoch": 0.4544502617801047, + "fcm_dpo/beta": 0.00493368785828352, + "fcm_dpo/delta": -0.02944992296397686, + "fcm_dpo/margin": 68.07068634033203, + "fcm_dpo/q_t": 0.4224163889884949, + "grad_norm": 73.08317565917969, + "learning_rate": 3.334948572847253e-07, + "logits/chosen": -0.8377935886383057, + "logits/rejected": -0.7964776754379272, + "logps/chosen": -411.5565185546875, + "logps/ref_chosen": -293.5498046875, + "logps/ref_rejected": -256.7830810546875, + "logps/rejected": -442.8605041503906, + "loss": 4.5865, + "margin_dpo/margin_mean": 68.07068634033203, + "margin_dpo/margin_std": 105.03534698486328, + "step": 217 + }, + { + "KL/chosen_KL_mean": -114.50249481201172, + "KL/mean": -144.8571319580078, + "KL/rejected_KL_mean": -175.2117462158203, + "KL/std": 92.68726348876953, + "epoch": 0.45654450261780105, + "fcm_dpo/beta": 0.0048797354102134705, + "fcm_dpo/delta": -0.008044907823204994, + "fcm_dpo/margin": 60.709266662597656, + "fcm_dpo/q_t": 0.4313804805278778, + "grad_norm": 64.33407592773438, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": -0.8685387372970581, + "logits/rejected": -0.8849347829818726, + "logps/chosen": -435.08184814453125, + "logps/ref_chosen": -320.579345703125, + "logps/ref_rejected": -294.0381164550781, + "logps/rejected": -469.2499084472656, + "loss": 4.6864, + "margin_dpo/margin_mean": 60.70926284790039, + "margin_dpo/margin_std": 99.89984130859375, + "step": 218 + }, + { + "KL/chosen_KL_mean": -108.34378051757812, + "KL/mean": -139.96334838867188, + "KL/rejected_KL_mean": -171.58291625976562, + "KL/std": 83.42886352539062, + "epoch": 0.4586387434554974, + "fcm_dpo/beta": 0.004817603155970573, + "fcm_dpo/delta": -0.010519558563828468, + "fcm_dpo/margin": 63.2391471862793, + "fcm_dpo/q_t": 0.4293725788593292, + "grad_norm": 86.39237976074219, + "learning_rate": 3.300347394584172e-07, + "logits/chosen": -0.847774863243103, + "logits/rejected": -0.8739175796508789, + "logps/chosen": -376.76239013671875, + "logps/ref_chosen": -268.4186096191406, + "logps/ref_rejected": -265.7808837890625, + "logps/rejected": -437.36376953125, + "loss": 4.6705, + "margin_dpo/margin_mean": 63.2391471862793, + "margin_dpo/margin_std": 100.10897827148438, + "step": 219 + }, + { + "KL/chosen_KL_mean": -109.47624206542969, + "KL/mean": -141.2556610107422, + "KL/rejected_KL_mean": -173.03509521484375, + "KL/std": 82.5869369506836, + "epoch": 0.4607329842931937, + "fcm_dpo/beta": 0.004888718016445637, + "fcm_dpo/delta": 0.03557638078927994, + "fcm_dpo/margin": 63.5588493347168, + "fcm_dpo/q_t": 0.4272876977920532, + "grad_norm": 79.07698822021484, + "learning_rate": 3.2829819606729477e-07, + "logits/chosen": -0.9154024124145508, + "logits/rejected": -0.8836403489112854, + "logps/chosen": -422.3627014160156, + "logps/ref_chosen": -312.8864440917969, + "logps/ref_rejected": -259.5191955566406, + "logps/rejected": -432.5542907714844, + "loss": 4.5985, + "margin_dpo/margin_mean": 63.55884552001953, + "margin_dpo/margin_std": 90.30359649658203, + "step": 220 + }, + { + "KL/chosen_KL_mean": -113.12982940673828, + "KL/mean": -138.3195037841797, + "KL/rejected_KL_mean": -163.50918579101562, + "KL/std": 86.43636322021484, + "epoch": 0.46282722513089003, + "fcm_dpo/beta": 0.004951215349137783, + "fcm_dpo/delta": -0.004505933728069067, + "fcm_dpo/margin": 50.37934112548828, + "fcm_dpo/q_t": 0.4432408809661865, + "grad_norm": 70.59790802001953, + "learning_rate": 3.265574537815398e-07, + "logits/chosen": -0.8173964023590088, + "logits/rejected": -0.8285130262374878, + "logps/chosen": -413.4556579589844, + "logps/ref_chosen": -300.32586669921875, + "logps/ref_rejected": -286.312255859375, + "logps/rejected": -449.8214111328125, + "loss": 4.8711, + "margin_dpo/margin_mean": 50.37934112548828, + "margin_dpo/margin_std": 101.99186706542969, + "step": 221 + }, + { + "KL/chosen_KL_mean": -106.68103790283203, + "KL/mean": -134.0705108642578, + "KL/rejected_KL_mean": -161.45997619628906, + "KL/std": 78.9898681640625, + "epoch": 0.4649214659685864, + "fcm_dpo/beta": 0.004972555674612522, + "fcm_dpo/delta": 0.014529617503285408, + "fcm_dpo/margin": 54.77894592285156, + "fcm_dpo/q_t": 0.43578585982322693, + "grad_norm": 83.3537826538086, + "learning_rate": 3.248126059518784e-07, + "logits/chosen": -0.9040961265563965, + "logits/rejected": -0.8702518939971924, + "logps/chosen": -403.79241943359375, + "logps/ref_chosen": -297.1113586425781, + "logps/ref_rejected": -235.53146362304688, + "logps/rejected": -396.991455078125, + "loss": 4.7331, + "margin_dpo/margin_mean": 54.7789421081543, + "margin_dpo/margin_std": 88.59034729003906, + "step": 222 + }, + { + "KL/chosen_KL_mean": -112.10569763183594, + "KL/mean": -143.2996826171875, + "KL/rejected_KL_mean": -174.49366760253906, + "KL/std": 86.74018096923828, + "epoch": 0.46701570680628274, + "fcm_dpo/beta": 0.005028898827731609, + "fcm_dpo/delta": 0.010601907968521118, + "fcm_dpo/margin": 62.38795852661133, + "fcm_dpo/q_t": 0.4269620180130005, + "grad_norm": 95.17221069335938, + "learning_rate": 3.230637461492043e-07, + "logits/chosen": -0.8466207385063171, + "logits/rejected": -0.8117715716362, + "logps/chosen": -398.52081298828125, + "logps/ref_chosen": -286.41510009765625, + "logps/ref_rejected": -241.1181640625, + "logps/rejected": -415.61181640625, + "loss": 4.6314, + "margin_dpo/margin_mean": 62.38795852661133, + "margin_dpo/margin_std": 97.02510070800781, + "step": 223 + }, + { + "KL/chosen_KL_mean": -103.6304702758789, + "KL/mean": -138.8751220703125, + "KL/rejected_KL_mean": -174.11976623535156, + "KL/std": 85.77777099609375, + "epoch": 0.46910994764397906, + "fcm_dpo/beta": 0.005118512082844973, + "fcm_dpo/delta": 0.005692493170499802, + "fcm_dpo/margin": 70.48930358886719, + "fcm_dpo/q_t": 0.4164232611656189, + "grad_norm": 69.0962905883789, + "learning_rate": 3.213109681595612e-07, + "logits/chosen": -0.8163408637046814, + "logits/rejected": -0.8316698670387268, + "logps/chosen": -353.1227722167969, + "logps/ref_chosen": -249.49234008789062, + "logps/ref_rejected": -233.10752868652344, + "logps/rejected": -407.2273254394531, + "loss": 4.4561, + "margin_dpo/margin_mean": 70.48930358886719, + "margin_dpo/margin_std": 91.14543151855469, + "step": 224 + }, + { + "KL/chosen_KL_mean": -123.72078704833984, + "KL/mean": -154.9224395751953, + "KL/rejected_KL_mean": -186.1240997314453, + "KL/std": 93.72308349609375, + "epoch": 0.4712041884816754, + "fcm_dpo/beta": 0.005051084794104099, + "fcm_dpo/delta": 0.011575451120734215, + "fcm_dpo/margin": 62.4033088684082, + "fcm_dpo/q_t": 0.42811909317970276, + "grad_norm": 72.46212005615234, + "learning_rate": 3.1955436597911315e-07, + "logits/chosen": -0.8486655354499817, + "logits/rejected": -0.821964681148529, + "logps/chosen": -435.57916259765625, + "logps/ref_chosen": -311.8583679199219, + "logps/ref_rejected": -336.8523864746094, + "logps/rejected": -522.9765014648438, + "loss": 4.6715, + "margin_dpo/margin_mean": 62.40331268310547, + "margin_dpo/margin_std": 105.1444091796875, + "step": 225 + }, + { + "KL/chosen_KL_mean": -112.75259399414062, + "KL/mean": -141.30508422851562, + "KL/rejected_KL_mean": -169.85757446289062, + "KL/std": 89.33605194091797, + "epoch": 0.4732984293193717, + "fcm_dpo/beta": 0.005086014047265053, + "fcm_dpo/delta": -0.0021250159479677677, + "fcm_dpo/margin": 57.104957580566406, + "fcm_dpo/q_t": 0.4333423972129822, + "grad_norm": 56.582576751708984, + "learning_rate": 3.1779403380910425e-07, + "logits/chosen": -0.8840992450714111, + "logits/rejected": -0.8711143136024475, + "logps/chosen": -364.9538269042969, + "logps/ref_chosen": -252.20123291015625, + "logps/ref_rejected": -254.41162109375, + "logps/rejected": -424.2691650390625, + "loss": 4.7484, + "margin_dpo/margin_mean": 57.104957580566406, + "margin_dpo/margin_std": 105.84939575195312, + "step": 226 + }, + { + "KL/chosen_KL_mean": -117.15921020507812, + "KL/mean": -151.5059814453125, + "KL/rejected_KL_mean": -185.85281372070312, + "KL/std": 86.63410949707031, + "epoch": 0.47539267015706804, + "fcm_dpo/beta": 0.0050234198570251465, + "fcm_dpo/delta": -0.021003495901823044, + "fcm_dpo/margin": 68.6935806274414, + "fcm_dpo/q_t": 0.41943684220314026, + "grad_norm": 72.25760650634766, + "learning_rate": 3.160300660508064e-07, + "logits/chosen": -0.8448896408081055, + "logits/rejected": -0.8412713408470154, + "logps/chosen": -402.4186706542969, + "logps/ref_chosen": -285.25946044921875, + "logps/ref_rejected": -261.3220520019531, + "logps/rejected": -447.1748352050781, + "loss": 4.5729, + "margin_dpo/margin_mean": 68.6935806274414, + "margin_dpo/margin_std": 106.60133361816406, + "step": 227 + }, + { + "KL/chosen_KL_mean": -108.93597412109375, + "KL/mean": -143.87466430664062, + "KL/rejected_KL_mean": -178.81336975097656, + "KL/std": 85.76429748535156, + "epoch": 0.4774869109947644, + "fcm_dpo/beta": 0.005010589491575956, + "fcm_dpo/delta": -0.023567285388708115, + "fcm_dpo/margin": 69.87741088867188, + "fcm_dpo/q_t": 0.4189654588699341, + "grad_norm": 60.705631256103516, + "learning_rate": 3.1426255730045695e-07, + "logits/chosen": -0.8538888692855835, + "logits/rejected": -0.8198533654212952, + "logps/chosen": -422.7547607421875, + "logps/ref_chosen": -313.81878662109375, + "logps/ref_rejected": -258.07061767578125, + "logps/rejected": -436.8840026855469, + "loss": 4.511, + "margin_dpo/margin_mean": 69.87741088867188, + "margin_dpo/margin_std": 96.2317886352539, + "step": 228 + }, + { + "KL/chosen_KL_mean": -128.9109344482422, + "KL/mean": -168.88983154296875, + "KL/rejected_KL_mean": -208.8687286376953, + "KL/std": 94.05682373046875, + "epoch": 0.47958115183246075, + "fcm_dpo/beta": 0.00481749139726162, + "fcm_dpo/delta": -0.03823835402727127, + "fcm_dpo/margin": 79.95777130126953, + "fcm_dpo/q_t": 0.4103056490421295, + "grad_norm": 84.75572967529297, + "learning_rate": 3.1249160234418644e-07, + "logits/chosen": -0.8427700400352478, + "logits/rejected": -0.8486419916152954, + "logps/chosen": -420.8816833496094, + "logps/ref_chosen": -291.9707946777344, + "logps/ref_rejected": -263.42059326171875, + "logps/rejected": -472.2893371582031, + "loss": 4.4112, + "margin_dpo/margin_mean": 79.95777893066406, + "margin_dpo/margin_std": 104.22063446044922, + "step": 229 + }, + { + "KL/chosen_KL_mean": -126.17520141601562, + "KL/mean": -160.9757843017578, + "KL/rejected_KL_mean": -195.77639770507812, + "KL/std": 99.210205078125, + "epoch": 0.4816753926701571, + "fcm_dpo/beta": 0.004781956318765879, + "fcm_dpo/delta": 0.01782013103365898, + "fcm_dpo/margin": 69.6011734008789, + "fcm_dpo/q_t": 0.42428159713745117, + "grad_norm": 75.2130355834961, + "learning_rate": 3.1071729615293424e-07, + "logits/chosen": -0.8815920948982239, + "logits/rejected": -0.8813921213150024, + "logps/chosen": -359.435302734375, + "logps/ref_chosen": -233.2601318359375, + "logps/ref_rejected": -238.922119140625, + "logps/rejected": -434.698486328125, + "loss": 4.6226, + "margin_dpo/margin_mean": 69.6011734008789, + "margin_dpo/margin_std": 115.32308197021484, + "step": 230 + }, + { + "KL/chosen_KL_mean": -137.8847198486328, + "KL/mean": -166.61566162109375, + "KL/rejected_KL_mean": -195.3466033935547, + "KL/std": 88.32843780517578, + "epoch": 0.4837696335078534, + "fcm_dpo/beta": 0.004834107588976622, + "fcm_dpo/delta": 0.005063319578766823, + "fcm_dpo/margin": 57.46186828613281, + "fcm_dpo/q_t": 0.4348789155483246, + "grad_norm": 71.28133392333984, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": -0.8586534261703491, + "logits/rejected": -0.8459566235542297, + "logps/chosen": -460.0399169921875, + "logps/ref_chosen": -322.1551818847656, + "logps/ref_rejected": -280.97613525390625, + "logps/rejected": -476.3227844238281, + "loss": 4.7507, + "margin_dpo/margin_mean": 57.461883544921875, + "margin_dpo/margin_std": 99.15318298339844, + "step": 231 + }, + { + "KL/chosen_KL_mean": -136.840576171875, + "KL/mean": -166.6940155029297, + "KL/rejected_KL_mean": -196.54745483398438, + "KL/std": 97.31487274169922, + "epoch": 0.48586387434554973, + "fcm_dpo/beta": 0.004791594110429287, + "fcm_dpo/delta": -0.025890890508890152, + "fcm_dpo/margin": 59.70685577392578, + "fcm_dpo/q_t": 0.43394574522972107, + "grad_norm": 77.55052947998047, + "learning_rate": 3.071590108427243e-07, + "logits/chosen": -0.833568811416626, + "logits/rejected": -0.8151888251304626, + "logps/chosen": -408.5843505859375, + "logps/ref_chosen": -271.7437744140625, + "logps/ref_rejected": -249.94981384277344, + "logps/rejected": -446.4972839355469, + "loss": 4.7829, + "margin_dpo/margin_mean": 59.70685958862305, + "margin_dpo/margin_std": 108.40725708007812, + "step": 232 + }, + { + "KL/chosen_KL_mean": -142.8009033203125, + "KL/mean": -177.56996154785156, + "KL/rejected_KL_mean": -212.33900451660156, + "KL/std": 95.32579803466797, + "epoch": 0.48795811518324606, + "fcm_dpo/beta": 0.004678625613451004, + "fcm_dpo/delta": -0.037981029599905014, + "fcm_dpo/margin": 69.5381088256836, + "fcm_dpo/q_t": 0.42466118931770325, + "grad_norm": 63.76099395751953, + "learning_rate": 3.05375222543809e-07, + "logits/chosen": -0.8736680746078491, + "logits/rejected": -0.8643991947174072, + "logps/chosen": -428.14324951171875, + "logps/ref_chosen": -285.3423156738281, + "logps/ref_rejected": -266.34320068359375, + "logps/rejected": -478.68218994140625, + "loss": 4.6045, + "margin_dpo/margin_mean": 69.5381088256836, + "margin_dpo/margin_std": 104.16007232666016, + "step": 233 + }, + { + "KL/chosen_KL_mean": -138.01681518554688, + "KL/mean": -173.01083374023438, + "KL/rejected_KL_mean": -208.0048370361328, + "KL/std": 98.38790130615234, + "epoch": 0.4900523560209424, + "fcm_dpo/beta": 0.004634845070540905, + "fcm_dpo/delta": 0.012818412855267525, + "fcm_dpo/margin": 69.98805236816406, + "fcm_dpo/q_t": 0.42441120743751526, + "grad_norm": 64.4365005493164, + "learning_rate": 3.035884646397637e-07, + "logits/chosen": -0.8443531394004822, + "logits/rejected": -0.8304785490036011, + "logps/chosen": -432.92254638671875, + "logps/ref_chosen": -294.9057312011719, + "logps/ref_rejected": -299.37054443359375, + "logps/rejected": -507.37542724609375, + "loss": 4.6463, + "margin_dpo/margin_mean": 69.98805236816406, + "margin_dpo/margin_std": 114.38517761230469, + "step": 234 + }, + { + "KL/chosen_KL_mean": -149.86141967773438, + "KL/mean": -180.7827606201172, + "KL/rejected_KL_mean": -211.7041015625, + "KL/std": 103.66209411621094, + "epoch": 0.49214659685863876, + "fcm_dpo/beta": 0.0047287228517234325, + "fcm_dpo/delta": 0.039121147245168686, + "fcm_dpo/margin": 61.84267044067383, + "fcm_dpo/q_t": 0.4319824278354645, + "grad_norm": 76.6629638671875, + "learning_rate": 3.017988329489923e-07, + "logits/chosen": -0.863947868347168, + "logits/rejected": -0.855870246887207, + "logps/chosen": -439.3590087890625, + "logps/ref_chosen": -289.49755859375, + "logps/ref_rejected": -247.55076599121094, + "logps/rejected": -459.2548522949219, + "loss": 4.7503, + "margin_dpo/margin_mean": 61.84267044067383, + "margin_dpo/margin_std": 115.53366088867188, + "step": 235 + }, + { + "KL/chosen_KL_mean": -145.44284057617188, + "KL/mean": -175.6923828125, + "KL/rejected_KL_mean": -205.94192504882812, + "KL/std": 100.11052703857422, + "epoch": 0.4942408376963351, + "fcm_dpo/beta": 0.004794456996023655, + "fcm_dpo/delta": -0.02471328340470791, + "fcm_dpo/margin": 60.49909591674805, + "fcm_dpo/q_t": 0.43273258209228516, + "grad_norm": 72.24032592773438, + "learning_rate": 3.000064234440111e-07, + "logits/chosen": -0.8775507807731628, + "logits/rejected": -0.8777625560760498, + "logps/chosen": -434.3274841308594, + "logps/ref_chosen": -288.8846435546875, + "logps/ref_rejected": -242.0452880859375, + "logps/rejected": -447.9872131347656, + "loss": 4.7912, + "margin_dpo/margin_mean": 60.49909210205078, + "margin_dpo/margin_std": 114.68053436279297, + "step": 236 + }, + { + "KL/chosen_KL_mean": -158.35903930664062, + "KL/mean": -189.57205200195312, + "KL/rejected_KL_mean": -220.7850341796875, + "KL/std": 89.05985260009766, + "epoch": 0.4963350785340314, + "fcm_dpo/beta": 0.004720585886389017, + "fcm_dpo/delta": 0.015051835216581821, + "fcm_dpo/margin": 62.426025390625, + "fcm_dpo/q_t": 0.4318310618400574, + "grad_norm": 90.479248046875, + "learning_rate": 2.9821133224630223e-07, + "logits/chosen": -0.8398151397705078, + "logits/rejected": -0.8189886212348938, + "logps/chosen": -423.8377685546875, + "logps/ref_chosen": -265.47869873046875, + "logps/ref_rejected": -267.9891357421875, + "logps/rejected": -488.774169921875, + "loss": 4.7226, + "margin_dpo/margin_mean": 62.426025390625, + "margin_dpo/margin_std": 109.10443115234375, + "step": 237 + }, + { + "KL/chosen_KL_mean": -163.52406311035156, + "KL/mean": -198.2044677734375, + "KL/rejected_KL_mean": -232.88482666015625, + "KL/std": 100.38282775878906, + "epoch": 0.49842931937172774, + "fcm_dpo/beta": 0.0047994209453463554, + "fcm_dpo/delta": 0.0028489064425230026, + "fcm_dpo/margin": 69.36079406738281, + "fcm_dpo/q_t": 0.42530369758605957, + "grad_norm": 77.99054718017578, + "learning_rate": 2.964136556211588e-07, + "logits/chosen": -0.8484768867492676, + "logits/rejected": -0.8160721659660339, + "logps/chosen": -475.5267333984375, + "logps/ref_chosen": -312.0026550292969, + "logps/ref_rejected": -270.0257263183594, + "logps/rejected": -502.9105224609375, + "loss": 4.6355, + "margin_dpo/margin_mean": 69.36080169677734, + "margin_dpo/margin_std": 117.8523941040039, + "step": 238 + }, + { + "KL/chosen_KL_mean": -161.76864624023438, + "KL/mean": -191.57293701171875, + "KL/rejected_KL_mean": -221.37716674804688, + "KL/std": 100.97264862060547, + "epoch": 0.5005235602094241, + "fcm_dpo/beta": 0.004840420559048653, + "fcm_dpo/delta": 0.02600950561463833, + "fcm_dpo/margin": 59.608524322509766, + "fcm_dpo/q_t": 0.4337758719921112, + "grad_norm": 65.61921691894531, + "learning_rate": 2.946134899725226e-07, + "logits/chosen": -0.8408248424530029, + "logits/rejected": -0.880803108215332, + "logps/chosen": -428.93585205078125, + "logps/ref_chosen": -267.167236328125, + "logps/ref_rejected": -275.99468994140625, + "logps/rejected": -497.37188720703125, + "loss": 4.8741, + "margin_dpo/margin_mean": 59.608524322509766, + "margin_dpo/margin_std": 132.66128540039062, + "step": 239 + }, + { + "KL/chosen_KL_mean": -158.13919067382812, + "KL/mean": -196.2527313232422, + "KL/rejected_KL_mean": -234.36627197265625, + "KL/std": 103.12637329101562, + "epoch": 0.5026178010471204, + "fcm_dpo/beta": 0.0048082321882247925, + "fcm_dpo/delta": -0.015010528266429901, + "fcm_dpo/margin": 76.22711181640625, + "fcm_dpo/q_t": 0.4153556823730469, + "grad_norm": 79.14559173583984, + "learning_rate": 2.9281093183781403e-07, + "logits/chosen": -0.9028453230857849, + "logits/rejected": -0.8958457708358765, + "logps/chosen": -444.1188049316406, + "logps/ref_chosen": -285.9796142578125, + "logps/ref_rejected": -256.8258056640625, + "logps/rejected": -491.1921081542969, + "loss": 4.4889, + "margin_dpo/margin_mean": 76.22711181640625, + "margin_dpo/margin_std": 109.41165924072266, + "step": 240 + }, + { + "KL/chosen_KL_mean": -180.80047607421875, + "KL/mean": -206.89230346679688, + "KL/rejected_KL_mean": -232.984130859375, + "KL/std": 111.80062866210938, + "epoch": 0.5047120418848168, + "fcm_dpo/beta": 0.004808627534657717, + "fcm_dpo/delta": -0.006103941705077887, + "fcm_dpo/margin": 52.18364715576172, + "fcm_dpo/q_t": 0.4433174133300781, + "grad_norm": 88.64216613769531, + "learning_rate": 2.910060778827554e-07, + "logits/chosen": -0.8175089955329895, + "logits/rejected": -0.7948691844940186, + "logps/chosen": -442.31732177734375, + "logps/ref_chosen": -261.516845703125, + "logps/ref_rejected": -250.2250518798828, + "logps/rejected": -483.20916748046875, + "loss": 4.9968, + "margin_dpo/margin_mean": 52.18364715576172, + "margin_dpo/margin_std": 127.83100128173828, + "step": 241 + }, + { + "KL/chosen_KL_mean": -161.50523376464844, + "KL/mean": -195.77635192871094, + "KL/rejected_KL_mean": -230.0474395751953, + "KL/std": 106.44923400878906, + "epoch": 0.506806282722513, + "fcm_dpo/beta": 0.004742825403809547, + "fcm_dpo/delta": -0.012836568988859653, + "fcm_dpo/margin": 68.5422134399414, + "fcm_dpo/q_t": 0.42594078183174133, + "grad_norm": 88.80677032470703, + "learning_rate": 2.891990248961871e-07, + "logits/chosen": -0.8814034461975098, + "logits/rejected": -0.856747031211853, + "logps/chosen": -432.01922607421875, + "logps/ref_chosen": -270.51397705078125, + "logps/ref_rejected": -244.8560791015625, + "logps/rejected": -474.9035339355469, + "loss": 4.6268, + "margin_dpo/margin_mean": 68.5422134399414, + "margin_dpo/margin_std": 111.54056549072266, + "step": 242 + }, + { + "KL/chosen_KL_mean": -153.52972412109375, + "KL/mean": -195.6244659423828, + "KL/rejected_KL_mean": -237.71920776367188, + "KL/std": 106.60342407226562, + "epoch": 0.5089005235602094, + "fcm_dpo/beta": 0.0047475434839725494, + "fcm_dpo/delta": -0.034220486879348755, + "fcm_dpo/margin": 84.18949127197266, + "fcm_dpo/q_t": 0.41172146797180176, + "grad_norm": 102.534912109375, + "learning_rate": 2.873898697848762e-07, + "logits/chosen": -0.8814060091972351, + "logits/rejected": -0.8652539849281311, + "logps/chosen": -478.21173095703125, + "logps/ref_chosen": -324.68206787109375, + "logps/ref_rejected": -307.1111755371094, + "logps/rejected": -544.8303833007812, + "loss": 4.4595, + "margin_dpo/margin_mean": 84.18949127197266, + "margin_dpo/margin_std": 128.6507110595703, + "step": 243 + }, + { + "KL/chosen_KL_mean": -152.66043090820312, + "KL/mean": -192.0305633544922, + "KL/rejected_KL_mean": -231.4007110595703, + "KL/std": 100.23159790039062, + "epoch": 0.5109947643979058, + "fcm_dpo/beta": 0.004584737122058868, + "fcm_dpo/delta": -0.017737431451678276, + "fcm_dpo/margin": 78.74027252197266, + "fcm_dpo/q_t": 0.4154585003852844, + "grad_norm": 77.94464111328125, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": -0.863549530506134, + "logits/rejected": -0.8185126185417175, + "logps/chosen": -471.6396789550781, + "logps/ref_chosen": -318.979248046875, + "logps/ref_rejected": -269.67572021484375, + "logps/rejected": -501.0764465332031, + "loss": 4.5167, + "margin_dpo/margin_mean": 78.74027252197266, + "margin_dpo/margin_std": 112.84671020507812, + "step": 244 + }, + { + "KL/chosen_KL_mean": -143.22264099121094, + "KL/mean": -182.23419189453125, + "KL/rejected_KL_mean": -221.24574279785156, + "KL/std": 96.40372467041016, + "epoch": 0.5130890052356021, + "fcm_dpo/beta": 0.004495944362133741, + "fcm_dpo/delta": -0.03692924976348877, + "fcm_dpo/margin": 78.02310943603516, + "fcm_dpo/q_t": 0.41797515749931335, + "grad_norm": 59.87691879272461, + "learning_rate": 2.837656413735479e-07, + "logits/chosen": -0.8684712648391724, + "logits/rejected": -0.8744341731071472, + "logps/chosen": -438.1206970214844, + "logps/ref_chosen": -294.8980712890625, + "logps/ref_rejected": -239.8111114501953, + "logps/rejected": -461.0568542480469, + "loss": 4.5402, + "margin_dpo/margin_mean": 78.02310943603516, + "margin_dpo/margin_std": 112.71392822265625, + "step": 245 + }, + { + "KL/chosen_KL_mean": -157.49560546875, + "KL/mean": -186.96458435058594, + "KL/rejected_KL_mean": -216.43356323242188, + "KL/std": 101.2094497680664, + "epoch": 0.5151832460732985, + "fcm_dpo/beta": 0.004423782229423523, + "fcm_dpo/delta": -0.0013894562143832445, + "fcm_dpo/margin": 58.93793487548828, + "fcm_dpo/q_t": 0.44051581621170044, + "grad_norm": 90.08110809326172, + "learning_rate": 2.8195076242990116e-07, + "logits/chosen": -0.8500803709030151, + "logits/rejected": -0.8542726039886475, + "logps/chosen": -438.1810302734375, + "logps/ref_chosen": -280.6854248046875, + "logps/ref_rejected": -253.65382385253906, + "logps/rejected": -470.0873718261719, + "loss": 4.8966, + "margin_dpo/margin_mean": 58.93793487548828, + "margin_dpo/margin_std": 127.81275177001953, + "step": 246 + }, + { + "KL/chosen_KL_mean": -140.9234619140625, + "KL/mean": -177.16494750976562, + "KL/rejected_KL_mean": -213.40638732910156, + "KL/std": 101.20616149902344, + "epoch": 0.5172774869109947, + "fcm_dpo/beta": 0.0044204313308000565, + "fcm_dpo/delta": -0.004756400361657143, + "fcm_dpo/margin": 72.48292541503906, + "fcm_dpo/q_t": 0.4261864423751831, + "grad_norm": 62.47346878051758, + "learning_rate": 2.801341700638307e-07, + "logits/chosen": -0.8428160548210144, + "logits/rejected": -0.8401237726211548, + "logps/chosen": -422.0325927734375, + "logps/ref_chosen": -281.1091003417969, + "logps/ref_rejected": -260.3700866699219, + "logps/rejected": -473.7765197753906, + "loss": 4.6164, + "margin_dpo/margin_mean": 72.48292541503906, + "margin_dpo/margin_std": 109.14017486572266, + "step": 247 + }, + { + "KL/chosen_KL_mean": -138.6498565673828, + "KL/mean": -173.0037384033203, + "KL/rejected_KL_mean": -207.35763549804688, + "KL/std": 101.77959442138672, + "epoch": 0.5193717277486911, + "fcm_dpo/beta": 0.004360491875559092, + "fcm_dpo/delta": -0.01580795831978321, + "fcm_dpo/margin": 68.70777893066406, + "fcm_dpo/q_t": 0.4298118054866791, + "grad_norm": 68.65995788574219, + "learning_rate": 2.7831596169367227e-07, + "logits/chosen": -0.813584566116333, + "logits/rejected": -0.8213151693344116, + "logps/chosen": -408.9682312011719, + "logps/ref_chosen": -270.318359375, + "logps/ref_rejected": -233.46778869628906, + "logps/rejected": -440.825439453125, + "loss": 4.6974, + "margin_dpo/margin_mean": 68.70777893066406, + "margin_dpo/margin_std": 112.36260986328125, + "step": 248 + }, + { + "KL/chosen_KL_mean": -151.7535400390625, + "KL/mean": -181.5082550048828, + "KL/rejected_KL_mean": -211.26295471191406, + "KL/std": 104.59439849853516, + "epoch": 0.5214659685863874, + "fcm_dpo/beta": 0.004381960723549128, + "fcm_dpo/delta": 0.020416682586073875, + "fcm_dpo/margin": 59.50940704345703, + "fcm_dpo/q_t": 0.4388493001461029, + "grad_norm": 62.078365325927734, + "learning_rate": 2.7649623482442274e-07, + "logits/chosen": -0.8454968333244324, + "logits/rejected": -0.8196409344673157, + "logps/chosen": -427.56243896484375, + "logps/ref_chosen": -275.8088684082031, + "logps/ref_rejected": -243.45138549804688, + "logps/rejected": -454.7143249511719, + "loss": 4.8702, + "margin_dpo/margin_mean": 59.50940704345703, + "margin_dpo/margin_std": 124.977783203125, + "step": 249 + }, + { + "KL/chosen_KL_mean": -146.8738250732422, + "KL/mean": -185.1446533203125, + "KL/rejected_KL_mean": -223.41546630859375, + "KL/std": 114.59971618652344, + "epoch": 0.5235602094240838, + "fcm_dpo/beta": 0.004331628791987896, + "fcm_dpo/delta": -0.035680629312992096, + "fcm_dpo/margin": 76.54167938232422, + "fcm_dpo/q_t": 0.4232044219970703, + "grad_norm": 52.845611572265625, + "learning_rate": 2.7467508704251135e-07, + "logits/chosen": -0.8485902547836304, + "logits/rejected": -0.8440597057342529, + "logps/chosen": -439.36834716796875, + "logps/ref_chosen": -292.4945373535156, + "logps/ref_rejected": -284.2869567871094, + "logps/rejected": -507.70245361328125, + "loss": 4.6384, + "margin_dpo/margin_mean": 76.54167938232422, + "margin_dpo/margin_std": 125.18535614013672, + "step": 250 + }, + { + "KL/chosen_KL_mean": -128.29568481445312, + "KL/mean": -163.9156951904297, + "KL/rejected_KL_mean": -199.53573608398438, + "KL/std": 107.50858306884766, + "epoch": 0.5256544502617801, + "fcm_dpo/beta": 0.004250232130289078, + "fcm_dpo/delta": -0.027963757514953613, + "fcm_dpo/margin": 71.24006652832031, + "fcm_dpo/q_t": 0.43231555819511414, + "grad_norm": 63.01066970825195, + "learning_rate": 2.7285261601056697e-07, + "logits/chosen": -0.8441932201385498, + "logits/rejected": -0.8291101455688477, + "logps/chosen": -410.03228759765625, + "logps/ref_chosen": -281.736572265625, + "logps/ref_rejected": -255.9419708251953, + "logps/rejected": -455.47772216796875, + "loss": 4.6987, + "margin_dpo/margin_mean": 71.24006652832031, + "margin_dpo/margin_std": 119.5110092163086, + "step": 251 + }, + { + "KL/chosen_KL_mean": -138.88409423828125, + "KL/mean": -173.60968017578125, + "KL/rejected_KL_mean": -208.3352508544922, + "KL/std": 102.1337661743164, + "epoch": 0.5277486910994764, + "fcm_dpo/beta": 0.004174236208200455, + "fcm_dpo/delta": -0.011869529262185097, + "fcm_dpo/margin": 69.45118713378906, + "fcm_dpo/q_t": 0.43159157037734985, + "grad_norm": 62.90778732299805, + "learning_rate": 2.7102891946217994e-07, + "logits/chosen": -0.8931678533554077, + "logits/rejected": -0.8665968179702759, + "logps/chosen": -434.85150146484375, + "logps/ref_chosen": -295.9674072265625, + "logps/ref_rejected": -280.111572265625, + "logps/rejected": -488.44683837890625, + "loss": 4.7147, + "margin_dpo/margin_mean": 69.45118713378906, + "margin_dpo/margin_std": 117.09326171875, + "step": 252 + }, + { + "KL/chosen_KL_mean": -144.9630584716797, + "KL/mean": -177.68711853027344, + "KL/rejected_KL_mean": -210.41119384765625, + "KL/std": 112.56256103515625, + "epoch": 0.5298429319371728, + "fcm_dpo/beta": 0.00418475829064846, + "fcm_dpo/delta": 0.008287381380796432, + "fcm_dpo/margin": 65.44812774658203, + "fcm_dpo/q_t": 0.4371293783187866, + "grad_norm": 56.794734954833984, + "learning_rate": 2.692040951966617e-07, + "logits/chosen": -0.8609364628791809, + "logits/rejected": -0.8533194661140442, + "logps/chosen": -422.0353088378906, + "logps/ref_chosen": -277.072265625, + "logps/ref_rejected": -247.31643676757812, + "logps/rejected": -457.7276306152344, + "loss": 4.812, + "margin_dpo/margin_mean": 65.44812774658203, + "margin_dpo/margin_std": 124.55648040771484, + "step": 253 + }, + { + "KL/chosen_KL_mean": -136.63442993164062, + "KL/mean": -172.9150390625, + "KL/rejected_KL_mean": -209.19570922851562, + "KL/std": 108.1693115234375, + "epoch": 0.5319371727748691, + "fcm_dpo/beta": 0.0041767992079257965, + "fcm_dpo/delta": -0.02163619175553322, + "fcm_dpo/margin": 72.561279296875, + "fcm_dpo/q_t": 0.430123507976532, + "grad_norm": 50.65597152709961, + "learning_rate": 2.6737824107379947e-07, + "logits/chosen": -0.8018378019332886, + "logits/rejected": -0.7838313579559326, + "logps/chosen": -406.5822448730469, + "logps/ref_chosen": -269.9478454589844, + "logps/ref_rejected": -249.45005798339844, + "logps/rejected": -458.645751953125, + "loss": 4.6683, + "margin_dpo/margin_mean": 72.56128692626953, + "margin_dpo/margin_std": 116.72087097167969, + "step": 254 + }, + { + "KL/chosen_KL_mean": -130.73049926757812, + "KL/mean": -171.3893280029297, + "KL/rejected_KL_mean": -212.04815673828125, + "KL/std": 108.50567626953125, + "epoch": 0.5340314136125655, + "fcm_dpo/beta": 0.004186869598925114, + "fcm_dpo/delta": 0.02620495669543743, + "fcm_dpo/margin": 81.31766510009766, + "fcm_dpo/q_t": 0.42155683040618896, + "grad_norm": 64.20274353027344, + "learning_rate": 2.655514550086086e-07, + "logits/chosen": -0.8257165551185608, + "logits/rejected": -0.792091429233551, + "logps/chosen": -437.38580322265625, + "logps/ref_chosen": -306.6552734375, + "logps/ref_rejected": -254.47528076171875, + "logps/rejected": -466.5234069824219, + "loss": 4.5829, + "margin_dpo/margin_mean": 81.31766510009766, + "margin_dpo/margin_std": 127.40750122070312, + "step": 255 + }, + { + "KL/chosen_KL_mean": -134.72642517089844, + "KL/mean": -171.73529052734375, + "KL/rejected_KL_mean": -208.74420166015625, + "KL/std": 108.54015350341797, + "epoch": 0.5361256544502618, + "fcm_dpo/beta": 0.004111228045076132, + "fcm_dpo/delta": -0.056310322135686874, + "fcm_dpo/margin": 74.01776123046875, + "fcm_dpo/q_t": 0.4271259903907776, + "grad_norm": 98.39281463623047, + "learning_rate": 2.6372383496608186e-07, + "logits/chosen": -0.85221928358078, + "logits/rejected": -0.8382247686386108, + "logps/chosen": -458.4446105957031, + "logps/ref_chosen": -323.7181701660156, + "logps/ref_rejected": -254.1871337890625, + "logps/rejected": -462.9313049316406, + "loss": 4.7255, + "margin_dpo/margin_mean": 74.01776123046875, + "margin_dpo/margin_std": 125.59674835205078, + "step": 256 + }, + { + "KL/chosen_KL_mean": -137.28506469726562, + "KL/mean": -176.9401092529297, + "KL/rejected_KL_mean": -216.59512329101562, + "KL/std": 103.74283599853516, + "epoch": 0.5382198952879581, + "fcm_dpo/beta": 0.004046399611979723, + "fcm_dpo/delta": 0.012473970651626587, + "fcm_dpo/margin": 79.31005859375, + "fcm_dpo/q_t": 0.4253496825695038, + "grad_norm": 80.60918426513672, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": -0.8426645994186401, + "logits/rejected": -0.8348857164382935, + "logps/chosen": -404.4971618652344, + "logps/ref_chosen": -267.21209716796875, + "logps/ref_rejected": -249.12579345703125, + "logps/rejected": -465.72088623046875, + "loss": 4.5916, + "margin_dpo/margin_mean": 79.31005859375, + "margin_dpo/margin_std": 119.5707778930664, + "step": 257 + }, + { + "KL/chosen_KL_mean": -145.31399536132812, + "KL/mean": -187.23464965820312, + "KL/rejected_KL_mean": -229.1552734375, + "KL/std": 105.5355224609375, + "epoch": 0.5403141361256545, + "fcm_dpo/beta": 0.003976684529334307, + "fcm_dpo/delta": -0.035320185124874115, + "fcm_dpo/margin": 83.84129333496094, + "fcm_dpo/q_t": 0.4211466908454895, + "grad_norm": 66.4306411743164, + "learning_rate": 2.600664850273538e-07, + "logits/chosen": -0.8702799677848816, + "logits/rejected": -0.8408257961273193, + "logps/chosen": -422.9967346191406, + "logps/ref_chosen": -277.6827392578125, + "logps/ref_rejected": -250.73385620117188, + "logps/rejected": -479.8891296386719, + "loss": 4.5294, + "margin_dpo/margin_mean": 83.8412857055664, + "margin_dpo/margin_std": 111.361328125, + "step": 258 + }, + { + "KL/chosen_KL_mean": -138.1536102294922, + "KL/mean": -175.6834716796875, + "KL/rejected_KL_mean": -213.21331787109375, + "KL/std": 108.60619354248047, + "epoch": 0.5424083769633508, + "fcm_dpo/beta": 0.003963233437389135, + "fcm_dpo/delta": 0.0012955646961927414, + "fcm_dpo/margin": 75.05972290039062, + "fcm_dpo/q_t": 0.4318161606788635, + "grad_norm": 80.89569091796875, + "learning_rate": 2.582369512637302e-07, + "logits/chosen": -0.8907499313354492, + "logits/rejected": -0.885222315788269, + "logps/chosen": -432.7635498046875, + "logps/ref_chosen": -294.6099853515625, + "logps/ref_rejected": -272.2725830078125, + "logps/rejected": -485.4858703613281, + "loss": 4.6869, + "margin_dpo/margin_mean": 75.05972290039062, + "margin_dpo/margin_std": 123.2925033569336, + "step": 259 + }, + { + "KL/chosen_KL_mean": -151.42819213867188, + "KL/mean": -169.2168426513672, + "KL/rejected_KL_mean": -187.00550842285156, + "KL/std": 95.43949127197266, + "epoch": 0.5445026178010471, + "fcm_dpo/beta": 0.003953398671001196, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 35.57732391357422, + "fcm_dpo/q_t": 0.4671649634838104, + "grad_norm": 74.01944732666016, + "learning_rate": 2.5640697577740815e-07, + "logits/chosen": -0.8799023628234863, + "logits/rejected": -0.873131513595581, + "logps/chosen": -442.2852783203125, + "logps/ref_chosen": -290.85711669921875, + "logps/ref_rejected": -277.5970153808594, + "logps/rejected": -464.6025390625, + "loss": 5.2063, + "margin_dpo/margin_mean": 35.57732009887695, + "margin_dpo/margin_std": 112.50765228271484, + "step": 260 + }, + { + "KL/chosen_KL_mean": -163.0951690673828, + "KL/mean": -195.61669921875, + "KL/rejected_KL_mean": -228.1382293701172, + "KL/std": 109.09551239013672, + "epoch": 0.5465968586387434, + "fcm_dpo/beta": 0.003954585175961256, + "fcm_dpo/delta": 0.004022921435534954, + "fcm_dpo/margin": 65.04306030273438, + "fcm_dpo/q_t": 0.44060125946998596, + "grad_norm": 52.7375373840332, + "learning_rate": 2.5457665670441937e-07, + "logits/chosen": -0.778756856918335, + "logits/rejected": -0.787772536277771, + "logps/chosen": -414.2274169921875, + "logps/ref_chosen": -251.13223266601562, + "logps/ref_rejected": -244.76016235351562, + "logps/rejected": -472.89837646484375, + "loss": 4.8579, + "margin_dpo/margin_mean": 65.04306030273438, + "margin_dpo/margin_std": 130.55770874023438, + "step": 261 + }, + { + "KL/chosen_KL_mean": -157.522216796875, + "KL/mean": -198.93121337890625, + "KL/rejected_KL_mean": -240.34019470214844, + "KL/std": 108.54368591308594, + "epoch": 0.5486910994764398, + "fcm_dpo/beta": 0.003956255502998829, + "fcm_dpo/delta": -0.019527418538928032, + "fcm_dpo/margin": 82.81796264648438, + "fcm_dpo/q_t": 0.4236336648464203, + "grad_norm": 64.078125, + "learning_rate": 2.527460921992209e-07, + "logits/chosen": -0.8164616823196411, + "logits/rejected": -0.8031435012817383, + "logps/chosen": -457.2440185546875, + "logps/ref_chosen": -299.7217712402344, + "logps/ref_rejected": -277.0969543457031, + "logps/rejected": -517.4371948242188, + "loss": 4.5771, + "margin_dpo/margin_mean": 82.81796264648438, + "margin_dpo/margin_std": 120.71702575683594, + "step": 262 + }, + { + "KL/chosen_KL_mean": -167.35940551757812, + "KL/mean": -204.05819702148438, + "KL/rejected_KL_mean": -240.75698852539062, + "KL/std": 112.38304138183594, + "epoch": 0.5507853403141362, + "fcm_dpo/beta": 0.0039150831289589405, + "fcm_dpo/delta": 0.004613391123712063, + "fcm_dpo/margin": 73.39756774902344, + "fcm_dpo/q_t": 0.4326966404914856, + "grad_norm": 57.92847442626953, + "learning_rate": 2.509153804294318e-07, + "logits/chosen": -0.8228314518928528, + "logits/rejected": -0.8032459020614624, + "logps/chosen": -447.3119812011719, + "logps/ref_chosen": -279.95257568359375, + "logps/ref_rejected": -256.5327453613281, + "logps/rejected": -497.28973388671875, + "loss": 4.7408, + "margin_dpo/margin_mean": 73.39756774902344, + "margin_dpo/margin_std": 127.13219451904297, + "step": 263 + }, + { + "KL/chosen_KL_mean": -162.80029296875, + "KL/mean": -206.62985229492188, + "KL/rejected_KL_mean": -250.45938110351562, + "KL/std": 113.47503662109375, + "epoch": 0.5528795811518324, + "fcm_dpo/beta": 0.003873462090268731, + "fcm_dpo/delta": -0.020922593772411346, + "fcm_dpo/margin": 87.65908813476562, + "fcm_dpo/q_t": 0.42167770862579346, + "grad_norm": 74.20809936523438, + "learning_rate": 2.4908461957056825e-07, + "logits/chosen": -0.8265196084976196, + "logits/rejected": -0.8174469470977783, + "logps/chosen": -423.3354187011719, + "logps/ref_chosen": -260.53509521484375, + "logps/ref_rejected": -255.53799438476562, + "logps/rejected": -505.9973449707031, + "loss": 4.5581, + "margin_dpo/margin_mean": 87.65908813476562, + "margin_dpo/margin_std": 128.83778381347656, + "step": 264 + }, + { + "KL/chosen_KL_mean": -166.21475219726562, + "KL/mean": -212.19488525390625, + "KL/rejected_KL_mean": -258.17498779296875, + "KL/std": 123.5406265258789, + "epoch": 0.5549738219895288, + "fcm_dpo/beta": 0.0038373975548893213, + "fcm_dpo/delta": -0.007137385196983814, + "fcm_dpo/margin": 91.96025848388672, + "fcm_dpo/q_t": 0.4191870093345642, + "grad_norm": 64.72450256347656, + "learning_rate": 2.4725390780077905e-07, + "logits/chosen": -0.8913218379020691, + "logits/rejected": -0.8904725909233093, + "logps/chosen": -449.9277648925781, + "logps/ref_chosen": -283.7130432128906, + "logps/ref_rejected": -270.3209533691406, + "logps/rejected": -528.4959716796875, + "loss": 4.5638, + "margin_dpo/margin_mean": 91.96025848388672, + "margin_dpo/margin_std": 144.87387084960938, + "step": 265 + }, + { + "KL/chosen_KL_mean": -152.7468719482422, + "KL/mean": -199.51541137695312, + "KL/rejected_KL_mean": -246.28395080566406, + "KL/std": 112.15851593017578, + "epoch": 0.5570680628272251, + "fcm_dpo/beta": 0.0038449072744697332, + "fcm_dpo/delta": -0.0055340006947517395, + "fcm_dpo/margin": 93.53707885742188, + "fcm_dpo/q_t": 0.41636204719543457, + "grad_norm": 73.08821868896484, + "learning_rate": 2.454233432955807e-07, + "logits/chosen": -0.9144778847694397, + "logits/rejected": -0.8848609328269958, + "logps/chosen": -430.84613037109375, + "logps/ref_chosen": -278.09930419921875, + "logps/ref_rejected": -260.6734619140625, + "logps/rejected": -506.9573974609375, + "loss": 4.4522, + "margin_dpo/margin_mean": 93.53707885742188, + "margin_dpo/margin_std": 120.03593444824219, + "step": 266 + }, + { + "KL/chosen_KL_mean": -182.0021514892578, + "KL/mean": -217.25442504882812, + "KL/rejected_KL_mean": -252.5067138671875, + "KL/std": 117.42387390136719, + "epoch": 0.5591623036649215, + "fcm_dpo/beta": 0.003810013411566615, + "fcm_dpo/delta": -0.003673842176795006, + "fcm_dpo/margin": 70.50455474853516, + "fcm_dpo/q_t": 0.4374660849571228, + "grad_norm": 74.9322738647461, + "learning_rate": 2.435930242225919e-07, + "logits/chosen": -0.8645923733711243, + "logits/rejected": -0.8707435131072998, + "logps/chosen": -462.3353271484375, + "logps/ref_chosen": -280.33319091796875, + "logps/ref_rejected": -247.78099060058594, + "logps/rejected": -500.28765869140625, + "loss": 4.8086, + "margin_dpo/margin_mean": 70.50454711914062, + "margin_dpo/margin_std": 132.0673065185547, + "step": 267 + }, + { + "KL/chosen_KL_mean": -172.97525024414062, + "KL/mean": -217.17364501953125, + "KL/rejected_KL_mean": -261.3720703125, + "KL/std": 120.18418884277344, + "epoch": 0.5612565445026177, + "fcm_dpo/beta": 0.0038873879238963127, + "fcm_dpo/delta": 0.034165654331445694, + "fcm_dpo/margin": 88.39683532714844, + "fcm_dpo/q_t": 0.42039138078689575, + "grad_norm": 72.54732513427734, + "learning_rate": 2.4176304873626984e-07, + "logits/chosen": -0.829031229019165, + "logits/rejected": -0.807607889175415, + "logps/chosen": -477.1539611816406, + "logps/ref_chosen": -304.1787109375, + "logps/ref_rejected": -272.80316162109375, + "logps/rejected": -534.1752319335938, + "loss": 4.5311, + "margin_dpo/margin_mean": 88.39683532714844, + "margin_dpo/margin_std": 126.67704010009766, + "step": 268 + }, + { + "KL/chosen_KL_mean": -176.60040283203125, + "KL/mean": -205.09878540039062, + "KL/rejected_KL_mean": -233.59716796875, + "KL/std": 104.05487060546875, + "epoch": 0.5633507853403141, + "fcm_dpo/beta": 0.003917751833796501, + "fcm_dpo/delta": 0.0035603800788521767, + "fcm_dpo/margin": 56.99675750732422, + "fcm_dpo/q_t": 0.4470731317996979, + "grad_norm": 71.9229965209961, + "learning_rate": 2.399335149726463e-07, + "logits/chosen": -0.8421822786331177, + "logits/rejected": -0.835168719291687, + "logps/chosen": -426.4455261230469, + "logps/ref_chosen": -249.84512329101562, + "logps/ref_rejected": -223.37356567382812, + "logps/rejected": -456.9707336425781, + "loss": 4.9429, + "margin_dpo/margin_mean": 56.996761322021484, + "margin_dpo/margin_std": 125.362548828125, + "step": 269 + }, + { + "KL/chosen_KL_mean": -185.8561248779297, + "KL/mean": -224.6792755126953, + "KL/rejected_KL_mean": -263.5024108886719, + "KL/std": 120.34756469726562, + "epoch": 0.5654450261780105, + "fcm_dpo/beta": 0.003944146912544966, + "fcm_dpo/delta": 0.01949712075293064, + "fcm_dpo/margin": 77.64627838134766, + "fcm_dpo/q_t": 0.43020257353782654, + "grad_norm": 67.843505859375, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": -0.934403657913208, + "logits/rejected": -0.9376578330993652, + "logps/chosen": -504.4185485839844, + "logps/ref_chosen": -318.5623779296875, + "logps/ref_rejected": -281.1880798339844, + "logps/rejected": -544.6904296875, + "loss": 4.7418, + "margin_dpo/margin_mean": 77.64627838134766, + "margin_dpo/margin_std": 149.40438842773438, + "step": 270 + }, + { + "KL/chosen_KL_mean": -181.69509887695312, + "KL/mean": -218.57923889160156, + "KL/rejected_KL_mean": -255.46336364746094, + "KL/std": 120.86827850341797, + "epoch": 0.5675392670157068, + "fcm_dpo/beta": 0.003990632947534323, + "fcm_dpo/delta": 0.0031273523345589638, + "fcm_dpo/margin": 73.76829528808594, + "fcm_dpo/q_t": 0.4324439764022827, + "grad_norm": 58.05919647216797, + "learning_rate": 2.3627616503391812e-07, + "logits/chosen": -0.7795909643173218, + "logits/rejected": -0.7716021537780762, + "logps/chosen": -465.7997741699219, + "logps/ref_chosen": -284.104736328125, + "logps/ref_rejected": -253.9580535888672, + "logps/rejected": -509.42144775390625, + "loss": 4.7087, + "margin_dpo/margin_mean": 73.76829528808594, + "margin_dpo/margin_std": 125.9541015625, + "step": 271 + }, + { + "KL/chosen_KL_mean": -169.88497924804688, + "KL/mean": -210.8214111328125, + "KL/rejected_KL_mean": -251.75787353515625, + "KL/std": 119.86404418945312, + "epoch": 0.5696335078534032, + "fcm_dpo/beta": 0.0039629037491977215, + "fcm_dpo/delta": -0.0006313333287835121, + "fcm_dpo/margin": 81.87287902832031, + "fcm_dpo/q_t": 0.42572107911109924, + "grad_norm": 86.46068572998047, + "learning_rate": 2.344485449913914e-07, + "logits/chosen": -0.883471667766571, + "logits/rejected": -0.8656666874885559, + "logps/chosen": -467.2440185546875, + "logps/ref_chosen": -297.3590087890625, + "logps/ref_rejected": -279.20196533203125, + "logps/rejected": -530.9597778320312, + "loss": 4.678, + "margin_dpo/margin_mean": 81.87287902832031, + "margin_dpo/margin_std": 145.54736328125, + "step": 272 + }, + { + "KL/chosen_KL_mean": -174.3992919921875, + "KL/mean": -216.345947265625, + "KL/rejected_KL_mean": -258.2926025390625, + "KL/std": 115.58932495117188, + "epoch": 0.5717277486910994, + "fcm_dpo/beta": 0.003995942883193493, + "fcm_dpo/delta": 0.009679936803877354, + "fcm_dpo/margin": 83.89334106445312, + "fcm_dpo/q_t": 0.4239039421081543, + "grad_norm": 88.02112579345703, + "learning_rate": 2.3262175892620062e-07, + "logits/chosen": -0.8663466572761536, + "logits/rejected": -0.8779019117355347, + "logps/chosen": -467.60504150390625, + "logps/ref_chosen": -293.20574951171875, + "logps/ref_rejected": -274.7646789550781, + "logps/rejected": -533.0572509765625, + "loss": 4.6362, + "margin_dpo/margin_mean": 83.89334869384766, + "margin_dpo/margin_std": 143.1727752685547, + "step": 273 + }, + { + "KL/chosen_KL_mean": -172.38304138183594, + "KL/mean": -221.9813690185547, + "KL/rejected_KL_mean": -271.5797119140625, + "KL/std": 125.11678314208984, + "epoch": 0.5738219895287958, + "fcm_dpo/beta": 0.0040048351511359215, + "fcm_dpo/delta": 0.0025398635771125555, + "fcm_dpo/margin": 99.1966552734375, + "fcm_dpo/q_t": 0.4098568260669708, + "grad_norm": 81.96195220947266, + "learning_rate": 2.3079590480333827e-07, + "logits/chosen": -0.8212272524833679, + "logits/rejected": -0.78861403465271, + "logps/chosen": -442.94171142578125, + "logps/ref_chosen": -270.55865478515625, + "logps/ref_rejected": -239.47048950195312, + "logps/rejected": -511.0502014160156, + "loss": 4.4372, + "margin_dpo/margin_mean": 99.19667053222656, + "margin_dpo/margin_std": 144.70790100097656, + "step": 274 + }, + { + "KL/chosen_KL_mean": -157.3403778076172, + "KL/mean": -207.17703247070312, + "KL/rejected_KL_mean": -257.013671875, + "KL/std": 118.32302856445312, + "epoch": 0.5759162303664922, + "fcm_dpo/beta": 0.003956434316933155, + "fcm_dpo/delta": -0.030254080891609192, + "fcm_dpo/margin": 99.67333221435547, + "fcm_dpo/q_t": 0.40914809703826904, + "grad_norm": 58.78046417236328, + "learning_rate": 2.2897108053782e-07, + "logits/chosen": -0.875052809715271, + "logits/rejected": -0.8559980988502502, + "logps/chosen": -407.65960693359375, + "logps/ref_chosen": -250.31922912597656, + "logps/ref_rejected": -249.3187255859375, + "logps/rejected": -506.3324279785156, + "loss": 4.3663, + "margin_dpo/margin_mean": 99.67330932617188, + "margin_dpo/margin_std": 125.57425689697266, + "step": 275 + }, + { + "KL/chosen_KL_mean": -171.91647338867188, + "KL/mean": -217.97848510742188, + "KL/rejected_KL_mean": -264.04046630859375, + "KL/std": 126.98786926269531, + "epoch": 0.5780104712041885, + "fcm_dpo/beta": 0.0038329358212649822, + "fcm_dpo/delta": -0.035872187465429306, + "fcm_dpo/margin": 92.1240005493164, + "fcm_dpo/q_t": 0.4182504713535309, + "grad_norm": 55.775657653808594, + "learning_rate": 2.2714738398943308e-07, + "logits/chosen": -0.9235668182373047, + "logits/rejected": -0.9017929434776306, + "logps/chosen": -469.54754638671875, + "logps/ref_chosen": -297.6310729980469, + "logps/ref_rejected": -295.225830078125, + "logps/rejected": -559.2662963867188, + "loss": 4.556, + "margin_dpo/margin_mean": 92.1240005493164, + "margin_dpo/margin_std": 140.80569458007812, + "step": 276 + }, + { + "KL/chosen_KL_mean": -193.89830017089844, + "KL/mean": -225.9439697265625, + "KL/rejected_KL_mean": -257.9896240234375, + "KL/std": 126.7256088256836, + "epoch": 0.5801047120418849, + "fcm_dpo/beta": 0.003798598423600197, + "fcm_dpo/delta": -0.014602910727262497, + "fcm_dpo/margin": 64.09132385253906, + "fcm_dpo/q_t": 0.4443640112876892, + "grad_norm": 64.50138092041016, + "learning_rate": 2.2532491295748865e-07, + "logits/chosen": -0.8662960529327393, + "logits/rejected": -0.8660153746604919, + "logps/chosen": -460.2587890625, + "logps/ref_chosen": -266.3604736328125, + "logps/ref_rejected": -253.36767578125, + "logps/rejected": -511.3572998046875, + "loss": 4.9367, + "margin_dpo/margin_mean": 64.09132385253906, + "margin_dpo/margin_std": 143.8057861328125, + "step": 277 + }, + { + "KL/chosen_KL_mean": -202.2888641357422, + "KL/mean": -230.39736938476562, + "KL/rejected_KL_mean": -258.5058898925781, + "KL/std": 130.2160186767578, + "epoch": 0.5821989528795811, + "fcm_dpo/beta": 0.0037986235693097115, + "fcm_dpo/delta": 0.011226484552025795, + "fcm_dpo/margin": 56.217010498046875, + "fcm_dpo/q_t": 0.4543144702911377, + "grad_norm": 96.25747680664062, + "learning_rate": 2.2350376517557726e-07, + "logits/chosen": -0.8956958651542664, + "logits/rejected": -0.8570727109909058, + "logps/chosen": -469.6961669921875, + "logps/ref_chosen": -267.40728759765625, + "logps/ref_rejected": -229.5758514404297, + "logps/rejected": -488.08172607421875, + "loss": 5.1072, + "margin_dpo/margin_mean": 56.217010498046875, + "margin_dpo/margin_std": 157.5133514404297, + "step": 278 + }, + { + "KL/chosen_KL_mean": -185.69024658203125, + "KL/mean": -241.31065368652344, + "KL/rejected_KL_mean": -296.9310607910156, + "KL/std": 138.99148559570312, + "epoch": 0.5842931937172775, + "fcm_dpo/beta": 0.0037318726535886526, + "fcm_dpo/delta": -0.05649794638156891, + "fcm_dpo/margin": 111.24081420898438, + "fcm_dpo/q_t": 0.405769944190979, + "grad_norm": 66.29354858398438, + "learning_rate": 2.2168403830632769e-07, + "logits/chosen": -0.8187253475189209, + "logits/rejected": -0.8022856116294861, + "logps/chosen": -499.05804443359375, + "logps/ref_chosen": -313.3677978515625, + "logps/ref_rejected": -299.1744384765625, + "logps/rejected": -596.10546875, + "loss": 4.3506, + "margin_dpo/margin_mean": 111.24081420898438, + "margin_dpo/margin_std": 146.99954223632812, + "step": 279 + }, + { + "KL/chosen_KL_mean": -182.4557342529297, + "KL/mean": -221.92530822753906, + "KL/rejected_KL_mean": -261.3948669433594, + "KL/std": 114.89640045166016, + "epoch": 0.5863874345549738, + "fcm_dpo/beta": 0.0036900462582707405, + "fcm_dpo/delta": 0.03023211471736431, + "fcm_dpo/margin": 78.93912506103516, + "fcm_dpo/q_t": 0.43237319588661194, + "grad_norm": 65.89375305175781, + "learning_rate": 2.1986582993616925e-07, + "logits/chosen": -0.8743321895599365, + "logits/rejected": -0.8858817219734192, + "logps/chosen": -448.0115661621094, + "logps/ref_chosen": -265.5558166503906, + "logps/ref_rejected": -247.1573944091797, + "logps/rejected": -508.55224609375, + "loss": 4.7364, + "margin_dpo/margin_mean": 78.93912506103516, + "margin_dpo/margin_std": 143.53533935546875, + "step": 280 + }, + { + "KL/chosen_KL_mean": -207.59323120117188, + "KL/mean": -253.11624145507812, + "KL/rejected_KL_mean": -298.63922119140625, + "KL/std": 127.79008483886719, + "epoch": 0.5884816753926702, + "fcm_dpo/beta": 0.0036901538260281086, + "fcm_dpo/delta": -0.028504468500614166, + "fcm_dpo/margin": 91.0459976196289, + "fcm_dpo/q_t": 0.4240657687187195, + "grad_norm": 67.67650604248047, + "learning_rate": 2.1804923757009882e-07, + "logits/chosen": -0.8543596267700195, + "logits/rejected": -0.8635410070419312, + "logps/chosen": -502.892822265625, + "logps/ref_chosen": -295.2995910644531, + "logps/ref_rejected": -293.80877685546875, + "logps/rejected": -592.447998046875, + "loss": 4.6098, + "margin_dpo/margin_mean": 91.04598236083984, + "margin_dpo/margin_std": 144.34996032714844, + "step": 281 + }, + { + "KL/chosen_KL_mean": -202.552490234375, + "KL/mean": -242.41110229492188, + "KL/rejected_KL_mean": -282.2697448730469, + "KL/std": 125.7529067993164, + "epoch": 0.5905759162303665, + "fcm_dpo/beta": 0.003655828535556793, + "fcm_dpo/delta": 0.0036571845412254333, + "fcm_dpo/margin": 79.71723937988281, + "fcm_dpo/q_t": 0.43203699588775635, + "grad_norm": 72.81260681152344, + "learning_rate": 2.1623435862645205e-07, + "logits/chosen": -0.8544440269470215, + "logits/rejected": -0.8522729873657227, + "logps/chosen": -521.1896362304688, + "logps/ref_chosen": -318.63714599609375, + "logps/ref_rejected": -273.5943603515625, + "logps/rejected": -555.8641357421875, + "loss": 4.7281, + "margin_dpo/margin_mean": 79.71723937988281, + "margin_dpo/margin_std": 139.59954833984375, + "step": 282 + }, + { + "KL/chosen_KL_mean": -202.61456298828125, + "KL/mean": -244.5399932861328, + "KL/rejected_KL_mean": -286.4654541015625, + "KL/std": 124.06411743164062, + "epoch": 0.5926701570680628, + "fcm_dpo/beta": 0.0036506266333162785, + "fcm_dpo/delta": -0.0035746542271226645, + "fcm_dpo/margin": 83.85090637207031, + "fcm_dpo/q_t": 0.43061453104019165, + "grad_norm": 57.49915313720703, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": -0.876941442489624, + "logits/rejected": -0.8653864860534668, + "logps/chosen": -457.2751159667969, + "logps/ref_chosen": -254.66053771972656, + "logps/ref_rejected": -236.8627166748047, + "logps/rejected": -523.3281860351562, + "loss": 4.6959, + "margin_dpo/margin_mean": 83.85089111328125, + "margin_dpo/margin_std": 145.12371826171875, + "step": 283 + }, + { + "KL/chosen_KL_mean": -210.0465545654297, + "KL/mean": -254.60189819335938, + "KL/rejected_KL_mean": -299.1572265625, + "KL/std": 129.64718627929688, + "epoch": 0.5947643979057592, + "fcm_dpo/beta": 0.0035921428352594376, + "fcm_dpo/delta": -0.02668537199497223, + "fcm_dpo/margin": 89.11066436767578, + "fcm_dpo/q_t": 0.42657744884490967, + "grad_norm": 73.3239517211914, + "learning_rate": 2.1261013021512378e-07, + "logits/chosen": -0.8376402258872986, + "logits/rejected": -0.8031928539276123, + "logps/chosen": -483.4017639160156, + "logps/ref_chosen": -273.355224609375, + "logps/ref_rejected": -259.84759521484375, + "logps/rejected": -559.0048217773438, + "loss": 4.6655, + "margin_dpo/margin_mean": 89.11066436767578, + "margin_dpo/margin_std": 149.5060577392578, + "step": 284 + }, + { + "KL/chosen_KL_mean": -217.2931365966797, + "KL/mean": -246.5795135498047, + "KL/rejected_KL_mean": -275.86590576171875, + "KL/std": 122.36019897460938, + "epoch": 0.5968586387434555, + "fcm_dpo/beta": 0.0035818694159388542, + "fcm_dpo/delta": 0.003046293742954731, + "fcm_dpo/margin": 58.572792053222656, + "fcm_dpo/q_t": 0.4514738619327545, + "grad_norm": 84.60891723632812, + "learning_rate": 2.1080097510381294e-07, + "logits/chosen": -0.8543550372123718, + "logits/rejected": -0.847979724407196, + "logps/chosen": -527.0953369140625, + "logps/ref_chosen": -309.8022155761719, + "logps/ref_rejected": -279.11846923828125, + "logps/rejected": -554.984375, + "loss": 5.0642, + "margin_dpo/margin_mean": 58.572792053222656, + "margin_dpo/margin_std": 150.84938049316406, + "step": 285 + }, + { + "KL/chosen_KL_mean": -208.0546112060547, + "KL/mean": -246.29867553710938, + "KL/rejected_KL_mean": -284.5427551269531, + "KL/std": 131.73287963867188, + "epoch": 0.5989528795811518, + "fcm_dpo/beta": 0.00358386617153883, + "fcm_dpo/delta": 0.004873398691415787, + "fcm_dpo/margin": 76.48812103271484, + "fcm_dpo/q_t": 0.43887919187545776, + "grad_norm": 92.04210662841797, + "learning_rate": 2.089939221172446e-07, + "logits/chosen": -0.8385077118873596, + "logits/rejected": -0.8326424360275269, + "logps/chosen": -479.5201721191406, + "logps/ref_chosen": -271.4655456542969, + "logps/ref_rejected": -279.531494140625, + "logps/rejected": -564.07421875, + "loss": 4.83, + "margin_dpo/margin_mean": 76.48812103271484, + "margin_dpo/margin_std": 154.2538604736328, + "step": 286 + }, + { + "KL/chosen_KL_mean": -207.88980102539062, + "KL/mean": -244.87991333007812, + "KL/rejected_KL_mean": -281.87005615234375, + "KL/std": 124.23129272460938, + "epoch": 0.6010471204188481, + "fcm_dpo/beta": 0.003638399764895439, + "fcm_dpo/delta": 0.026291579008102417, + "fcm_dpo/margin": 73.98025512695312, + "fcm_dpo/q_t": 0.4376610815525055, + "grad_norm": 68.3472900390625, + "learning_rate": 2.0718906816218595e-07, + "logits/chosen": -0.8518833518028259, + "logits/rejected": -0.8357435464859009, + "logps/chosen": -484.9830322265625, + "logps/ref_chosen": -277.0932312011719, + "logps/ref_rejected": -233.55599975585938, + "logps/rejected": -515.426025390625, + "loss": 4.859, + "margin_dpo/margin_mean": 73.98026275634766, + "margin_dpo/margin_std": 155.37527465820312, + "step": 287 + }, + { + "KL/chosen_KL_mean": -199.41354370117188, + "KL/mean": -238.12754821777344, + "KL/rejected_KL_mean": -276.84161376953125, + "KL/std": 129.22042846679688, + "epoch": 0.6031413612565445, + "fcm_dpo/beta": 0.003688263474032283, + "fcm_dpo/delta": -0.005784134846180677, + "fcm_dpo/margin": 77.42807006835938, + "fcm_dpo/q_t": 0.43434223532676697, + "grad_norm": 77.68144226074219, + "learning_rate": 2.053865100274774e-07, + "logits/chosen": -0.8734298944473267, + "logits/rejected": -0.8790357708930969, + "logps/chosen": -492.58172607421875, + "logps/ref_chosen": -293.1681823730469, + "logps/ref_rejected": -263.4059143066406, + "logps/rejected": -540.2474975585938, + "loss": 4.7674, + "margin_dpo/margin_mean": 77.4280776977539, + "margin_dpo/margin_std": 144.46633911132812, + "step": 288 + }, + { + "KL/chosen_KL_mean": -219.11532592773438, + "KL/mean": -241.2076873779297, + "KL/rejected_KL_mean": -263.3000183105469, + "KL/std": 120.56001281738281, + "epoch": 0.6052356020942409, + "fcm_dpo/beta": 0.00366892758756876, + "fcm_dpo/delta": 0.015712561085820198, + "fcm_dpo/margin": 44.1847038269043, + "fcm_dpo/q_t": 0.46217232942581177, + "grad_norm": 70.22594451904297, + "learning_rate": 2.035863443788411e-07, + "logits/chosen": -0.8561855554580688, + "logits/rejected": -0.8383423089981079, + "logps/chosen": -549.07275390625, + "logps/ref_chosen": -329.9574279785156, + "logps/ref_rejected": -276.7565002441406, + "logps/rejected": -540.0565185546875, + "loss": 5.1915, + "margin_dpo/margin_mean": 44.1847038269043, + "margin_dpo/margin_std": 136.41799926757812, + "step": 289 + }, + { + "KL/chosen_KL_mean": -193.29393005371094, + "KL/mean": -233.5602569580078, + "KL/rejected_KL_mean": -273.82659912109375, + "KL/std": 127.18586730957031, + "epoch": 0.6073298429319371, + "fcm_dpo/beta": 0.0036051059141755104, + "fcm_dpo/delta": -0.03102608397603035, + "fcm_dpo/margin": 80.5326919555664, + "fcm_dpo/q_t": 0.43328824639320374, + "grad_norm": 100.93278503417969, + "learning_rate": 2.0178866775369774e-07, + "logits/chosen": -0.8621543049812317, + "logits/rejected": -0.7959156036376953, + "logps/chosen": -517.9629516601562, + "logps/ref_chosen": -324.6690673828125, + "logps/ref_rejected": -311.8439636230469, + "logps/rejected": -585.6705932617188, + "loss": 4.7497, + "margin_dpo/margin_mean": 80.5326919555664, + "margin_dpo/margin_std": 144.66436767578125, + "step": 290 + }, + { + "KL/chosen_KL_mean": -185.8102264404297, + "KL/mean": -235.31077575683594, + "KL/rejected_KL_mean": -284.8113098144531, + "KL/std": 127.50725555419922, + "epoch": 0.6094240837696335, + "fcm_dpo/beta": 0.003556034294888377, + "fcm_dpo/delta": -0.027959764003753662, + "fcm_dpo/margin": 99.00109100341797, + "fcm_dpo/q_t": 0.41859322786331177, + "grad_norm": 57.9321174621582, + "learning_rate": 1.9999357655598891e-07, + "logits/chosen": -0.8408191204071045, + "logits/rejected": -0.8317016959190369, + "logps/chosen": -459.95428466796875, + "logps/ref_chosen": -274.1440734863281, + "logps/ref_rejected": -278.07208251953125, + "logps/rejected": -562.8833618164062, + "loss": 4.5248, + "margin_dpo/margin_mean": 99.00109100341797, + "margin_dpo/margin_std": 140.4064178466797, + "step": 291 + }, + { + "KL/chosen_KL_mean": -173.9268798828125, + "KL/mean": -216.89697265625, + "KL/rejected_KL_mean": -259.8670349121094, + "KL/std": 126.9601821899414, + "epoch": 0.6115183246073298, + "fcm_dpo/beta": 0.0034741119015961885, + "fcm_dpo/delta": -0.024277152493596077, + "fcm_dpo/margin": 85.94014739990234, + "fcm_dpo/q_t": 0.4312286972999573, + "grad_norm": 64.79742431640625, + "learning_rate": 1.9820116705100775e-07, + "logits/chosen": -0.8414206504821777, + "logits/rejected": -0.8373555541038513, + "logps/chosen": -433.2906188964844, + "logps/ref_chosen": -259.3636779785156, + "logps/ref_rejected": -279.30218505859375, + "logps/rejected": -539.1692504882812, + "loss": 4.7164, + "margin_dpo/margin_mean": 85.94014739990234, + "margin_dpo/margin_std": 148.6029815673828, + "step": 292 + }, + { + "KL/chosen_KL_mean": -196.158203125, + "KL/mean": -242.4661102294922, + "KL/rejected_KL_mean": -288.7740783691406, + "KL/std": 135.72119140625, + "epoch": 0.6136125654450262, + "fcm_dpo/beta": 0.003440189640969038, + "fcm_dpo/delta": -0.006982447579503059, + "fcm_dpo/margin": 92.61585998535156, + "fcm_dpo/q_t": 0.42598748207092285, + "grad_norm": 62.069820404052734, + "learning_rate": 1.9641153536023642e-07, + "logits/chosen": -0.9244207143783569, + "logits/rejected": -0.8833814859390259, + "logps/chosen": -499.92901611328125, + "logps/ref_chosen": -303.77081298828125, + "logps/ref_rejected": -270.07513427734375, + "logps/rejected": -558.8491821289062, + "loss": 4.6122, + "margin_dpo/margin_mean": 92.61585998535156, + "margin_dpo/margin_std": 139.9814910888672, + "step": 293 + }, + { + "KL/chosen_KL_mean": -186.78228759765625, + "KL/mean": -225.4820556640625, + "KL/rejected_KL_mean": -264.18182373046875, + "KL/std": 131.13441467285156, + "epoch": 0.6157068062827226, + "fcm_dpo/beta": 0.003445146605372429, + "fcm_dpo/delta": -0.002983933314681053, + "fcm_dpo/margin": 77.39950561523438, + "fcm_dpo/q_t": 0.4392993748188019, + "grad_norm": 56.854644775390625, + "learning_rate": 1.9462477745619106e-07, + "logits/chosen": -0.8256325125694275, + "logits/rejected": -0.8303864002227783, + "logps/chosen": -427.02056884765625, + "logps/ref_chosen": -240.23831176757812, + "logps/ref_rejected": -229.187744140625, + "logps/rejected": -493.3695373535156, + "loss": 4.8415, + "margin_dpo/margin_mean": 77.39949798583984, + "margin_dpo/margin_std": 154.39572143554688, + "step": 294 + }, + { + "KL/chosen_KL_mean": -188.3880615234375, + "KL/mean": -226.49118041992188, + "KL/rejected_KL_mean": -264.59429931640625, + "KL/std": 112.24237060546875, + "epoch": 0.6178010471204188, + "fcm_dpo/beta": 0.003445256035774946, + "fcm_dpo/delta": 0.014424074441194534, + "fcm_dpo/margin": 76.20626068115234, + "fcm_dpo/q_t": 0.4396905303001404, + "grad_norm": 69.74424743652344, + "learning_rate": 1.928409891572757e-07, + "logits/chosen": -0.8317238688468933, + "logits/rejected": -0.8348393440246582, + "logps/chosen": -439.3977966308594, + "logps/ref_chosen": -251.00970458984375, + "logps/ref_rejected": -244.15142822265625, + "logps/rejected": -508.74578857421875, + "loss": 4.7987, + "margin_dpo/margin_mean": 76.20626068115234, + "margin_dpo/margin_std": 140.69558715820312, + "step": 295 + }, + { + "KL/chosen_KL_mean": -192.59164428710938, + "KL/mean": -244.67086791992188, + "KL/rejected_KL_mean": -296.7500915527344, + "KL/std": 131.72547912597656, + "epoch": 0.6198952879581152, + "fcm_dpo/beta": 0.0034456930588930845, + "fcm_dpo/delta": -0.02413531392812729, + "fcm_dpo/margin": 104.15848541259766, + "fcm_dpo/q_t": 0.417242169380188, + "grad_norm": 68.13108825683594, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": -0.8389942646026611, + "logits/rejected": -0.8055410981178284, + "logps/chosen": -486.47222900390625, + "logps/ref_chosen": -293.880615234375, + "logps/ref_rejected": -283.4175720214844, + "logps/rejected": -580.167724609375, + "loss": 4.539, + "margin_dpo/margin_mean": 104.15848541259766, + "margin_dpo/margin_std": 157.52789306640625, + "step": 296 + }, + { + "KL/chosen_KL_mean": -189.89254760742188, + "KL/mean": -227.05242919921875, + "KL/rejected_KL_mean": -264.21234130859375, + "KL/std": 118.02452087402344, + "epoch": 0.6219895287958115, + "fcm_dpo/beta": 0.003379724221304059, + "fcm_dpo/delta": -0.012697052210569382, + "fcm_dpo/margin": 74.31977844238281, + "fcm_dpo/q_t": 0.44128429889678955, + "grad_norm": 60.725318908691406, + "learning_rate": 1.8928270384706582e-07, + "logits/chosen": -0.9069033265113831, + "logits/rejected": -0.9000098705291748, + "logps/chosen": -479.35260009765625, + "logps/ref_chosen": -289.4600830078125, + "logps/ref_rejected": -283.69110107421875, + "logps/rejected": -547.9033813476562, + "loss": 4.8268, + "margin_dpo/margin_mean": 74.31977844238281, + "margin_dpo/margin_std": 135.9732666015625, + "step": 297 + }, + { + "KL/chosen_KL_mean": -180.1241455078125, + "KL/mean": -226.821044921875, + "KL/rejected_KL_mean": -273.5179443359375, + "KL/std": 125.18611907958984, + "epoch": 0.6240837696335079, + "fcm_dpo/beta": 0.0033408179879188538, + "fcm_dpo/delta": -0.020929735153913498, + "fcm_dpo/margin": 93.3937759399414, + "fcm_dpo/q_t": 0.4280543327331543, + "grad_norm": 89.00557708740234, + "learning_rate": 1.875083976558136e-07, + "logits/chosen": -0.8199465870857239, + "logits/rejected": -0.8056501150131226, + "logps/chosen": -486.6391296386719, + "logps/ref_chosen": -306.5150146484375, + "logps/ref_rejected": -280.6969909667969, + "logps/rejected": -554.21484375, + "loss": 4.6812, + "margin_dpo/margin_mean": 93.3937759399414, + "margin_dpo/margin_std": 156.3482666015625, + "step": 298 + }, + { + "KL/chosen_KL_mean": -210.533447265625, + "KL/mean": -247.99322509765625, + "KL/rejected_KL_mean": -285.4530334472656, + "KL/std": 129.0129852294922, + "epoch": 0.6261780104712041, + "fcm_dpo/beta": 0.0033021531999111176, + "fcm_dpo/delta": -0.0064115277491509914, + "fcm_dpo/margin": 74.91960906982422, + "fcm_dpo/q_t": 0.442142128944397, + "grad_norm": 70.9146957397461, + "learning_rate": 1.8573744269954297e-07, + "logits/chosen": -0.8116376399993896, + "logits/rejected": -0.7935671210289001, + "logps/chosen": -491.8971862792969, + "logps/ref_chosen": -281.36376953125, + "logps/ref_rejected": -270.39508056640625, + "logps/rejected": -555.84814453125, + "loss": 4.8285, + "margin_dpo/margin_mean": 74.91960906982422, + "margin_dpo/margin_std": 130.8252410888672, + "step": 299 + }, + { + "KL/chosen_KL_mean": -212.4322509765625, + "KL/mean": -248.7073211669922, + "KL/rejected_KL_mean": -284.982421875, + "KL/std": 135.9843292236328, + "epoch": 0.6282722513089005, + "fcm_dpo/beta": 0.00330313085578382, + "fcm_dpo/delta": 0.010395125485956669, + "fcm_dpo/margin": 72.5501480102539, + "fcm_dpo/q_t": 0.4443802237510681, + "grad_norm": 66.76209259033203, + "learning_rate": 1.839699339491937e-07, + "logits/chosen": -0.836089551448822, + "logits/rejected": -0.8139994144439697, + "logps/chosen": -527.26806640625, + "logps/ref_chosen": -314.83575439453125, + "logps/ref_rejected": -269.1154479980469, + "logps/rejected": -554.0978393554688, + "loss": 4.9016, + "margin_dpo/margin_mean": 72.5501480102539, + "margin_dpo/margin_std": 154.16453552246094, + "step": 300 + }, + { + "KL/chosen_KL_mean": -201.32894897460938, + "KL/mean": -238.62901306152344, + "KL/rejected_KL_mean": -275.9290771484375, + "KL/std": 128.91531372070312, + "epoch": 0.6303664921465969, + "fcm_dpo/beta": 0.003320886753499508, + "fcm_dpo/delta": -0.009585805237293243, + "fcm_dpo/margin": 74.60012817382812, + "fcm_dpo/q_t": 0.4424639344215393, + "grad_norm": 81.5901870727539, + "learning_rate": 1.8220596619089573e-07, + "logits/chosen": -0.8685322999954224, + "logits/rejected": -0.8592681884765625, + "logps/chosen": -481.2235107421875, + "logps/ref_chosen": -279.89453125, + "logps/ref_rejected": -271.6694641113281, + "logps/rejected": -547.5985107421875, + "loss": 4.8592, + "margin_dpo/margin_mean": 74.60012817382812, + "margin_dpo/margin_std": 144.12062072753906, + "step": 301 + }, + { + "KL/chosen_KL_mean": -197.4744415283203, + "KL/mean": -246.68585205078125, + "KL/rejected_KL_mean": -295.89727783203125, + "KL/std": 135.5497589111328, + "epoch": 0.6324607329842932, + "fcm_dpo/beta": 0.0033070500940084457, + "fcm_dpo/delta": 0.005456558428704739, + "fcm_dpo/margin": 98.42284393310547, + "fcm_dpo/q_t": 0.42443469166755676, + "grad_norm": 70.69303131103516, + "learning_rate": 1.8044563402088682e-07, + "logits/chosen": -0.8222125172615051, + "logits/rejected": -0.8099421262741089, + "logps/chosen": -468.8062744140625, + "logps/ref_chosen": -271.3318176269531, + "logps/ref_rejected": -256.5587158203125, + "logps/rejected": -552.4559936523438, + "loss": 4.5808, + "margin_dpo/margin_mean": 98.42283630371094, + "margin_dpo/margin_std": 144.48379516601562, + "step": 302 + }, + { + "KL/chosen_KL_mean": -200.82537841796875, + "KL/mean": -243.748291015625, + "KL/rejected_KL_mean": -286.67120361328125, + "KL/std": 122.43389892578125, + "epoch": 0.6345549738219896, + "fcm_dpo/beta": 0.0033165114000439644, + "fcm_dpo/delta": 0.004780970048159361, + "fcm_dpo/margin": 85.84585571289062, + "fcm_dpo/q_t": 0.4328896403312683, + "grad_norm": 66.35929870605469, + "learning_rate": 1.7868903184043885e-07, + "logits/chosen": -0.8214887380599976, + "logits/rejected": -0.7972728610038757, + "logps/chosen": -505.7063903808594, + "logps/ref_chosen": -304.88104248046875, + "logps/ref_rejected": -269.063720703125, + "logps/rejected": -555.7349853515625, + "loss": 4.6998, + "margin_dpo/margin_mean": 85.84585571289062, + "margin_dpo/margin_std": 135.42276000976562, + "step": 303 + }, + { + "KL/chosen_KL_mean": -215.84197998046875, + "KL/mean": -262.8927001953125, + "KL/rejected_KL_mean": -309.9433898925781, + "KL/std": 149.1090850830078, + "epoch": 0.6366492146596858, + "fcm_dpo/beta": 0.003333859611302614, + "fcm_dpo/delta": 0.005845099687576294, + "fcm_dpo/margin": 94.10143280029297, + "fcm_dpo/q_t": 0.42921072244644165, + "grad_norm": 54.69770812988281, + "learning_rate": 1.7693625385079574e-07, + "logits/chosen": -0.8201622366905212, + "logits/rejected": -0.8260871171951294, + "logps/chosen": -506.55291748046875, + "logps/ref_chosen": -290.7109680175781, + "logps/ref_rejected": -237.6885986328125, + "logps/rejected": -547.6320190429688, + "loss": 4.677, + "margin_dpo/margin_mean": 94.10143280029297, + "margin_dpo/margin_std": 163.25584411621094, + "step": 304 + }, + { + "KL/chosen_KL_mean": -180.0332489013672, + "KL/mean": -250.61903381347656, + "KL/rejected_KL_mean": -321.204833984375, + "KL/std": 140.91082763671875, + "epoch": 0.6387434554973822, + "fcm_dpo/beta": 0.0031983419321477413, + "fcm_dpo/delta": -0.1049448549747467, + "fcm_dpo/margin": 141.17156982421875, + "fcm_dpo/q_t": 0.3961653709411621, + "grad_norm": 65.096923828125, + "learning_rate": 1.7518739404812155e-07, + "logits/chosen": -0.8557301759719849, + "logits/rejected": -0.8239875435829163, + "logps/chosen": -436.51727294921875, + "logps/ref_chosen": -256.4839782714844, + "logps/ref_rejected": -266.4063415527344, + "logps/rejected": -587.6111450195312, + "loss": 4.2208, + "margin_dpo/margin_mean": 141.17156982421875, + "margin_dpo/margin_std": 163.42698669433594, + "step": 305 + }, + { + "KL/chosen_KL_mean": -197.95596313476562, + "KL/mean": -238.4625244140625, + "KL/rejected_KL_mean": -278.9691162109375, + "KL/std": 122.23258209228516, + "epoch": 0.6408376963350786, + "fcm_dpo/beta": 0.0031039295718073845, + "fcm_dpo/delta": 0.02336675301194191, + "fcm_dpo/margin": 81.01317596435547, + "fcm_dpo/q_t": 0.4409124553203583, + "grad_norm": 70.48225402832031, + "learning_rate": 1.7344254621846017e-07, + "logits/chosen": -0.8595868945121765, + "logits/rejected": -0.8387668132781982, + "logps/chosen": -518.6051635742188, + "logps/ref_chosen": -320.6492004394531, + "logps/ref_rejected": -273.36773681640625, + "logps/rejected": -552.3368530273438, + "loss": 4.7914, + "margin_dpo/margin_mean": 81.01317596435547, + "margin_dpo/margin_std": 134.45468139648438, + "step": 306 + }, + { + "KL/chosen_KL_mean": -226.18310546875, + "KL/mean": -269.3814392089844, + "KL/rejected_KL_mean": -312.57977294921875, + "KL/std": 138.75106811523438, + "epoch": 0.6429319371727749, + "fcm_dpo/beta": 0.00315967109054327, + "fcm_dpo/delta": 0.011747002601623535, + "fcm_dpo/margin": 86.39667510986328, + "fcm_dpo/q_t": 0.4350773096084595, + "grad_norm": 107.51629638671875, + "learning_rate": 1.717018039327053e-07, + "logits/chosen": -0.797162652015686, + "logits/rejected": -0.8184522986412048, + "logps/chosen": -505.6372985839844, + "logps/ref_chosen": -279.4541931152344, + "logps/ref_rejected": -240.3796844482422, + "logps/rejected": -552.95947265625, + "loss": 4.7194, + "margin_dpo/margin_mean": 86.39667510986328, + "margin_dpo/margin_std": 132.84042358398438, + "step": 307 + }, + { + "KL/chosen_KL_mean": -245.1857147216797, + "KL/mean": -275.275146484375, + "KL/rejected_KL_mean": -305.36456298828125, + "KL/std": 142.4264373779297, + "epoch": 0.6450261780104712, + "fcm_dpo/beta": 0.003162857610732317, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 60.1788330078125, + "fcm_dpo/q_t": 0.4557945728302002, + "grad_norm": 66.74758911132812, + "learning_rate": 1.699652605415828e-07, + "logits/chosen": -0.8476360440254211, + "logits/rejected": -0.8570371866226196, + "logps/chosen": -541.7840576171875, + "logps/ref_chosen": -296.598388671875, + "logps/ref_rejected": -258.6953430175781, + "logps/rejected": -564.0598754882812, + "loss": 5.0646, + "margin_dpo/margin_mean": 60.1788330078125, + "margin_dpo/margin_std": 152.62615966796875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -229.7421112060547, + "KL/mean": -281.80914306640625, + "KL/rejected_KL_mean": -333.8762512207031, + "KL/std": 138.70144653320312, + "epoch": 0.6471204188481675, + "fcm_dpo/beta": 0.0031696951482445, + "fcm_dpo/delta": 0.008564796298742294, + "fcm_dpo/margin": 104.13411712646484, + "fcm_dpo/q_t": 0.4243236780166626, + "grad_norm": 69.46084594726562, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": -0.8503900766372681, + "logits/rejected": -0.8593624234199524, + "logps/chosen": -511.1302185058594, + "logps/ref_chosen": -281.3881530761719, + "logps/ref_rejected": -262.458740234375, + "logps/rejected": -596.3349609375, + "loss": 4.583, + "margin_dpo/margin_mean": 104.13410949707031, + "margin_dpo/margin_std": 161.4528045654297, + "step": 309 + }, + { + "KL/chosen_KL_mean": -222.09640502929688, + "KL/mean": -270.9717712402344, + "KL/rejected_KL_mean": -319.84710693359375, + "KL/std": 141.6556396484375, + "epoch": 0.6492146596858639, + "fcm_dpo/beta": 0.003191425697878003, + "fcm_dpo/delta": 0.008555184118449688, + "fcm_dpo/margin": 97.75069427490234, + "fcm_dpo/q_t": 0.4272443950176239, + "grad_norm": 97.37775421142578, + "learning_rate": 1.6650514271527465e-07, + "logits/chosen": -0.8479313850402832, + "logits/rejected": -0.8224340081214905, + "logps/chosen": -501.2836608886719, + "logps/ref_chosen": -279.1872863769531, + "logps/ref_rejected": -261.8279724121094, + "logps/rejected": -581.675048828125, + "loss": 4.6234, + "margin_dpo/margin_mean": 97.75069427490234, + "margin_dpo/margin_std": 146.51434326171875, + "step": 310 + }, + { + "KL/chosen_KL_mean": -227.7700958251953, + "KL/mean": -280.1990661621094, + "KL/rejected_KL_mean": -332.6280212402344, + "KL/std": 150.89422607421875, + "epoch": 0.6513089005235602, + "fcm_dpo/beta": 0.0031548929400742054, + "fcm_dpo/delta": -0.02223263308405876, + "fcm_dpo/margin": 104.85791015625, + "fcm_dpo/q_t": 0.42464131116867065, + "grad_norm": 55.97704315185547, + "learning_rate": 1.647817538357072e-07, + "logits/chosen": -0.8462434411048889, + "logits/rejected": -0.8253843784332275, + "logps/chosen": -499.1682434082031, + "logps/ref_chosen": -271.39813232421875, + "logps/ref_rejected": -266.12701416015625, + "logps/rejected": -598.7550659179688, + "loss": 4.6293, + "margin_dpo/margin_mean": 104.85791015625, + "margin_dpo/margin_std": 171.730224609375, + "step": 311 + }, + { + "KL/chosen_KL_mean": -225.2976837158203, + "KL/mean": -271.5206604003906, + "KL/rejected_KL_mean": -317.74365234375, + "KL/std": 148.50408935546875, + "epoch": 0.6534031413612565, + "fcm_dpo/beta": 0.0031063989736139774, + "fcm_dpo/delta": -0.02017890475690365, + "fcm_dpo/margin": 92.44593811035156, + "fcm_dpo/q_t": 0.4358768165111542, + "grad_norm": 69.38884735107422, + "learning_rate": 1.6306293495205755e-07, + "logits/chosen": -0.8498347997665405, + "logits/rejected": -0.8207939863204956, + "logps/chosen": -507.6827087402344, + "logps/ref_chosen": -282.3850402832031, + "logps/ref_rejected": -246.35389709472656, + "logps/rejected": -564.0975341796875, + "loss": 4.8437, + "margin_dpo/margin_mean": 92.44593048095703, + "margin_dpo/margin_std": 191.8460693359375, + "step": 312 + }, + { + "KL/chosen_KL_mean": -228.06698608398438, + "KL/mean": -272.4660949707031, + "KL/rejected_KL_mean": -316.86517333984375, + "KL/std": 152.6967315673828, + "epoch": 0.6554973821989529, + "fcm_dpo/beta": 0.0031254414934664965, + "fcm_dpo/delta": -0.0015552723780274391, + "fcm_dpo/margin": 88.7981948852539, + "fcm_dpo/q_t": 0.43964749574661255, + "grad_norm": 53.70850372314453, + "learning_rate": 1.6134877823936607e-07, + "logits/chosen": -0.894835352897644, + "logits/rejected": -0.8788830637931824, + "logps/chosen": -531.6978759765625, + "logps/ref_chosen": -303.630859375, + "logps/ref_rejected": -273.1156921386719, + "logps/rejected": -589.9808349609375, + "loss": 4.844, + "margin_dpo/margin_mean": 88.7981948852539, + "margin_dpo/margin_std": 178.9306640625, + "step": 313 + }, + { + "KL/chosen_KL_mean": -218.431640625, + "KL/mean": -264.6053466796875, + "KL/rejected_KL_mean": -310.779052734375, + "KL/std": 135.10806274414062, + "epoch": 0.6575916230366492, + "fcm_dpo/beta": 0.0031253828201442957, + "fcm_dpo/delta": 0.024184562265872955, + "fcm_dpo/margin": 92.347412109375, + "fcm_dpo/q_t": 0.43235987424850464, + "grad_norm": 63.3831901550293, + "learning_rate": 1.5963937562265522e-07, + "logits/chosen": -0.8976191282272339, + "logits/rejected": -0.8891264200210571, + "logps/chosen": -520.7359619140625, + "logps/ref_chosen": -302.3042907714844, + "logps/ref_rejected": -273.6416015625, + "logps/rejected": -584.4207153320312, + "loss": 4.7032, + "margin_dpo/margin_mean": 92.34739685058594, + "margin_dpo/margin_std": 150.6728057861328, + "step": 314 + }, + { + "KL/chosen_KL_mean": -211.95986938476562, + "KL/mean": -268.8974609375, + "KL/rejected_KL_mean": -325.8350524902344, + "KL/std": 141.79452514648438, + "epoch": 0.6596858638743456, + "fcm_dpo/beta": 0.0031580040231347084, + "fcm_dpo/delta": -0.00586454663425684, + "fcm_dpo/margin": 113.87518310546875, + "fcm_dpo/q_t": 0.4168047308921814, + "grad_norm": 71.96776580810547, + "learning_rate": 1.5793481877199943e-07, + "logits/chosen": -0.8908346891403198, + "logits/rejected": -0.8735600113868713, + "logps/chosen": -514.6891479492188, + "logps/ref_chosen": -302.729248046875, + "logps/ref_rejected": -270.26910400390625, + "logps/rejected": -596.1041259765625, + "loss": 4.4891, + "margin_dpo/margin_mean": 113.87518310546875, + "margin_dpo/margin_std": 158.78358459472656, + "step": 315 + }, + { + "KL/chosen_KL_mean": -222.13168334960938, + "KL/mean": -270.41119384765625, + "KL/rejected_KL_mean": -318.69073486328125, + "KL/std": 140.0904541015625, + "epoch": 0.6617801047120419, + "fcm_dpo/beta": 0.003170891897752881, + "fcm_dpo/delta": 0.0005847662687301636, + "fcm_dpo/margin": 96.55903625488281, + "fcm_dpo/q_t": 0.4291931390762329, + "grad_norm": 62.24413299560547, + "learning_rate": 1.562351990976095e-07, + "logits/chosen": -0.9054620265960693, + "logits/rejected": -0.8874289393424988, + "logps/chosen": -532.702392578125, + "logps/ref_chosen": -310.5706481933594, + "logps/ref_rejected": -272.9354553222656, + "logps/rejected": -591.626220703125, + "loss": 4.6598, + "margin_dpo/margin_mean": 96.55903625488281, + "margin_dpo/margin_std": 156.25230407714844, + "step": 316 + }, + { + "KL/chosen_KL_mean": -193.6438446044922, + "KL/mean": -238.2861328125, + "KL/rejected_KL_mean": -282.9284362792969, + "KL/std": 125.384033203125, + "epoch": 0.6638743455497382, + "fcm_dpo/beta": 0.0031651100143790245, + "fcm_dpo/delta": 0.015250356867909431, + "fcm_dpo/margin": 89.2845458984375, + "fcm_dpo/q_t": 0.43306419253349304, + "grad_norm": 67.99295043945312, + "learning_rate": 1.5454060774493065e-07, + "logits/chosen": -0.8674439787864685, + "logits/rejected": -0.8366461992263794, + "logps/chosen": -447.5442199707031, + "logps/ref_chosen": -253.90036010742188, + "logps/ref_rejected": -218.74078369140625, + "logps/rejected": -501.669189453125, + "loss": 4.68, + "margin_dpo/margin_mean": 89.2845458984375, + "margin_dpo/margin_std": 131.8083038330078, + "step": 317 + }, + { + "KL/chosen_KL_mean": -200.83892822265625, + "KL/mean": -259.72735595703125, + "KL/rejected_KL_mean": -318.61578369140625, + "KL/std": 138.72280883789062, + "epoch": 0.6659685863874345, + "fcm_dpo/beta": 0.0031195408664643764, + "fcm_dpo/delta": -0.029568474739789963, + "fcm_dpo/margin": 117.77685546875, + "fcm_dpo/q_t": 0.41461455821990967, + "grad_norm": 64.10457611083984, + "learning_rate": 1.5285113558975427e-07, + "logits/chosen": -0.9008802771568298, + "logits/rejected": -0.8715929388999939, + "logps/chosen": -471.66180419921875, + "logps/ref_chosen": -270.8228759765625, + "logps/ref_rejected": -255.30972290039062, + "logps/rejected": -573.925537109375, + "loss": 4.4471, + "margin_dpo/margin_mean": 117.77685546875, + "margin_dpo/margin_std": 154.478515625, + "step": 318 + }, + { + "KL/chosen_KL_mean": -211.363037109375, + "KL/mean": -264.632568359375, + "KL/rejected_KL_mean": -317.902099609375, + "KL/std": 134.839599609375, + "epoch": 0.6680628272251309, + "fcm_dpo/beta": 0.0031336878892034292, + "fcm_dpo/delta": 0.01707865111529827, + "fcm_dpo/margin": 106.53904724121094, + "fcm_dpo/q_t": 0.42066890001296997, + "grad_norm": 84.68042755126953, + "learning_rate": 1.5116687323334464e-07, + "logits/chosen": -0.8805448412895203, + "logits/rejected": -0.8484068512916565, + "logps/chosen": -512.3659057617188, + "logps/ref_chosen": -301.0028076171875, + "logps/ref_rejected": -242.39002990722656, + "logps/rejected": -560.2921142578125, + "loss": 4.506, + "margin_dpo/margin_mean": 106.53904724121094, + "margin_dpo/margin_std": 134.68338012695312, + "step": 319 + }, + { + "KL/chosen_KL_mean": -210.13380432128906, + "KL/mean": -259.4587707519531, + "KL/rejected_KL_mean": -308.78369140625, + "KL/std": 143.7012176513672, + "epoch": 0.6701570680628273, + "fcm_dpo/beta": 0.0031071542762219906, + "fcm_dpo/delta": -0.02263573370873928, + "fcm_dpo/margin": 98.64984893798828, + "fcm_dpo/q_t": 0.4299119710922241, + "grad_norm": 75.7774658203125, + "learning_rate": 1.4948791099758052e-07, + "logits/chosen": -0.8548473715782166, + "logits/rejected": -0.8449010848999023, + "logps/chosen": -513.7564086914062, + "logps/ref_chosen": -303.6225891113281, + "logps/ref_rejected": -280.85174560546875, + "logps/rejected": -589.6353759765625, + "loss": 4.6904, + "margin_dpo/margin_mean": 98.64985656738281, + "margin_dpo/margin_std": 171.7944793701172, + "step": 320 + }, + { + "KL/chosen_KL_mean": -231.89138793945312, + "KL/mean": -268.456787109375, + "KL/rejected_KL_mean": -305.02215576171875, + "KL/std": 141.75418090820312, + "epoch": 0.6722513089005235, + "fcm_dpo/beta": 0.0031160078942775726, + "fcm_dpo/delta": 0.01728174462914467, + "fcm_dpo/margin": 73.13074493408203, + "fcm_dpo/q_t": 0.4479866623878479, + "grad_norm": 60.38873291015625, + "learning_rate": 1.478143389201113e-07, + "logits/chosen": -0.865630567073822, + "logits/rejected": -0.8288436532020569, + "logps/chosen": -520.877197265625, + "logps/ref_chosen": -288.98583984375, + "logps/ref_rejected": -241.1822052001953, + "logps/rejected": -546.204345703125, + "loss": 4.9345, + "margin_dpo/margin_mean": 73.13074493408203, + "margin_dpo/margin_std": 156.70712280273438, + "step": 321 + }, + { + "KL/chosen_KL_mean": -239.51849365234375, + "KL/mean": -283.0441589355469, + "KL/rejected_KL_mean": -326.56982421875, + "KL/std": 145.20364379882812, + "epoch": 0.6743455497382199, + "fcm_dpo/beta": 0.0031907029915601015, + "fcm_dpo/delta": 0.04024318605661392, + "fcm_dpo/margin": 87.05138397216797, + "fcm_dpo/q_t": 0.4357774257659912, + "grad_norm": 65.08485412597656, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": -0.930508017539978, + "logits/rejected": -0.8933470845222473, + "logps/chosen": -548.0618896484375, + "logps/ref_chosen": -308.54345703125, + "logps/ref_rejected": -269.7995910644531, + "logps/rejected": -596.369384765625, + "loss": 4.8048, + "margin_dpo/margin_mean": 87.05138397216797, + "margin_dpo/margin_std": 171.3778076171875, + "step": 322 + }, + { + "KL/chosen_KL_mean": -233.23052978515625, + "KL/mean": -264.7674560546875, + "KL/rejected_KL_mean": -296.30438232421875, + "KL/std": 134.792236328125, + "epoch": 0.6764397905759162, + "fcm_dpo/beta": 0.003232902381569147, + "fcm_dpo/delta": -0.0018465555040165782, + "fcm_dpo/margin": 63.07388687133789, + "fcm_dpo/q_t": 0.4540364444255829, + "grad_norm": 92.90402221679688, + "learning_rate": 1.4448372394055246e-07, + "logits/chosen": -0.8878564834594727, + "logits/rejected": -0.8803995847702026, + "logps/chosen": -515.72412109375, + "logps/ref_chosen": -282.49365234375, + "logps/ref_rejected": -227.7105255126953, + "logps/rejected": -524.014892578125, + "loss": 5.0631, + "margin_dpo/margin_mean": 63.073883056640625, + "margin_dpo/margin_std": 164.00965881347656, + "step": 323 + }, + { + "KL/chosen_KL_mean": -203.592041015625, + "KL/mean": -258.118896484375, + "KL/rejected_KL_mean": -312.64569091796875, + "KL/std": 143.88446044921875, + "epoch": 0.6785340314136126, + "fcm_dpo/beta": 0.0031611351296305656, + "fcm_dpo/delta": -0.020262351259589195, + "fcm_dpo/margin": 109.05362701416016, + "fcm_dpo/q_t": 0.41922521591186523, + "grad_norm": 52.81629943847656, + "learning_rate": 1.428268596492364e-07, + "logits/chosen": -0.8115453124046326, + "logits/rejected": -0.8045988082885742, + "logps/chosen": -442.930419921875, + "logps/ref_chosen": -239.33836364746094, + "logps/ref_rejected": -230.53775024414062, + "logps/rejected": -543.1834716796875, + "loss": 4.5051, + "margin_dpo/margin_mean": 109.05361938476562, + "margin_dpo/margin_std": 144.5257568359375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -234.3408203125, + "KL/mean": -280.7336120605469, + "KL/rejected_KL_mean": -327.1264343261719, + "KL/std": 150.88209533691406, + "epoch": 0.680628272251309, + "fcm_dpo/beta": 0.003149219322949648, + "fcm_dpo/delta": -0.012197908945381641, + "fcm_dpo/margin": 92.78559875488281, + "fcm_dpo/q_t": 0.4342680275440216, + "grad_norm": 94.48786163330078, + "learning_rate": 1.4117574272818386e-07, + "logits/chosen": -0.8268110156059265, + "logits/rejected": -0.8105596303939819, + "logps/chosen": -514.9697875976562, + "logps/ref_chosen": -280.62896728515625, + "logps/ref_rejected": -270.5085754394531, + "logps/rejected": -597.6349487304688, + "loss": 4.8091, + "margin_dpo/margin_mean": 92.78559875488281, + "margin_dpo/margin_std": 185.6398468017578, + "step": 325 + }, + { + "KL/chosen_KL_mean": -214.26693725585938, + "KL/mean": -265.3741455078125, + "KL/rejected_KL_mean": -316.4813232421875, + "KL/std": 142.83740234375, + "epoch": 0.6827225130890052, + "fcm_dpo/beta": 0.0032117185182869434, + "fcm_dpo/delta": 0.028104102239012718, + "fcm_dpo/margin": 102.21440124511719, + "fcm_dpo/q_t": 0.4247254431247711, + "grad_norm": 64.72186279296875, + "learning_rate": 1.3953046172178413e-07, + "logits/chosen": -0.945293664932251, + "logits/rejected": -0.9358657598495483, + "logps/chosen": -455.2540588378906, + "logps/ref_chosen": -240.9871368408203, + "logps/ref_rejected": -261.0238342285156, + "logps/rejected": -577.5051879882812, + "loss": 4.6272, + "margin_dpo/margin_mean": 102.21440124511719, + "margin_dpo/margin_std": 167.8172149658203, + "step": 326 + }, + { + "KL/chosen_KL_mean": -207.91552734375, + "KL/mean": -264.1645812988281, + "KL/rejected_KL_mean": -320.4136657714844, + "KL/std": 134.82386779785156, + "epoch": 0.6848167539267016, + "fcm_dpo/beta": 0.0031878852751106024, + "fcm_dpo/delta": -0.03201831877231598, + "fcm_dpo/margin": 112.49812316894531, + "fcm_dpo/q_t": 0.4176490604877472, + "grad_norm": 94.11993408203125, + "learning_rate": 1.3789110486146468e-07, + "logits/chosen": -0.8883708119392395, + "logits/rejected": -0.8645241856575012, + "logps/chosen": -487.435546875, + "logps/ref_chosen": -279.52001953125, + "logps/ref_rejected": -269.51824951171875, + "logps/rejected": -589.931884765625, + "loss": 4.5059, + "margin_dpo/margin_mean": 112.49813079833984, + "margin_dpo/margin_std": 157.23403930664062, + "step": 327 + }, + { + "KL/chosen_KL_mean": -229.1092529296875, + "KL/mean": -280.052490234375, + "KL/rejected_KL_mean": -330.9957275390625, + "KL/std": 139.11770629882812, + "epoch": 0.6869109947643979, + "fcm_dpo/beta": 0.003118137363344431, + "fcm_dpo/delta": -0.0110612902790308, + "fcm_dpo/margin": 101.88642883300781, + "fcm_dpo/q_t": 0.42611968517303467, + "grad_norm": 63.17850112915039, + "learning_rate": 1.362577600609588e-07, + "logits/chosen": -0.8533894419670105, + "logits/rejected": -0.8511139154434204, + "logps/chosen": -530.1427612304688, + "logps/ref_chosen": -301.033447265625, + "logps/ref_rejected": -284.2101135253906, + "logps/rejected": -615.205810546875, + "loss": 4.6045, + "margin_dpo/margin_mean": 101.88642883300781, + "margin_dpo/margin_std": 151.49839782714844, + "step": 328 + }, + { + "KL/chosen_KL_mean": -243.52928161621094, + "KL/mean": -300.4892883300781, + "KL/rejected_KL_mean": -357.4493103027344, + "KL/std": 178.26580810546875, + "epoch": 0.6890052356020943, + "fcm_dpo/beta": 0.0030765547417104244, + "fcm_dpo/delta": -0.023377757519483566, + "fcm_dpo/margin": 113.92003631591797, + "fcm_dpo/q_t": 0.4244258999824524, + "grad_norm": 67.04179382324219, + "learning_rate": 1.3463051491159093e-07, + "logits/chosen": -0.8606047630310059, + "logits/rejected": -0.8284364938735962, + "logps/chosen": -563.5181274414062, + "logps/ref_chosen": -319.9888610839844, + "logps/ref_rejected": -307.5588684082031, + "logps/rejected": -665.0081787109375, + "loss": 4.6515, + "margin_dpo/margin_mean": 113.92002868652344, + "margin_dpo/margin_std": 201.48175048828125, + "step": 329 + }, + { + "KL/chosen_KL_mean": -238.3928985595703, + "KL/mean": -291.460205078125, + "KL/rejected_KL_mean": -344.5274963378906, + "KL/std": 156.5318603515625, + "epoch": 0.6910994764397905, + "fcm_dpo/beta": 0.0030223994981497526, + "fcm_dpo/delta": -0.023758048191666603, + "fcm_dpo/margin": 106.13461303710938, + "fcm_dpo/q_t": 0.42642486095428467, + "grad_norm": 73.99193572998047, + "learning_rate": 1.3300945667758012e-07, + "logits/chosen": -0.857183039188385, + "logits/rejected": -0.8630150556564331, + "logps/chosen": -539.5076904296875, + "logps/ref_chosen": -301.11474609375, + "logps/ref_rejected": -299.673095703125, + "logps/rejected": -644.2005615234375, + "loss": 4.6121, + "margin_dpo/margin_mean": 106.13459777832031, + "margin_dpo/margin_std": 160.67657470703125, + "step": 330 + }, + { + "KL/chosen_KL_mean": -226.96383666992188, + "KL/mean": -274.009765625, + "KL/rejected_KL_mean": -321.0557556152344, + "KL/std": 153.6221466064453, + "epoch": 0.6931937172774869, + "fcm_dpo/beta": 0.0030192735139280558, + "fcm_dpo/delta": 0.01395251415669918, + "fcm_dpo/margin": 94.09191131591797, + "fcm_dpo/q_t": 0.4358104467391968, + "grad_norm": 80.74335479736328, + "learning_rate": 1.3139467229135998e-07, + "logits/chosen": -0.8754410743713379, + "logits/rejected": -0.8539817333221436, + "logps/chosen": -504.5553283691406, + "logps/ref_chosen": -277.59149169921875, + "logps/ref_rejected": -256.025634765625, + "logps/rejected": -577.0813598632812, + "loss": 4.7924, + "margin_dpo/margin_mean": 94.09191131591797, + "margin_dpo/margin_std": 185.90357971191406, + "step": 331 + }, + { + "KL/chosen_KL_mean": -240.19212341308594, + "KL/mean": -288.86236572265625, + "KL/rejected_KL_mean": -337.5325927734375, + "KL/std": 150.13291931152344, + "epoch": 0.6952879581151833, + "fcm_dpo/beta": 0.003028081264346838, + "fcm_dpo/delta": -0.003697561100125313, + "fcm_dpo/margin": 97.34046936035156, + "fcm_dpo/q_t": 0.4329964518547058, + "grad_norm": 66.77661895751953, + "learning_rate": 1.2978624834891626e-07, + "logits/chosen": -0.8756115436553955, + "logits/rejected": -0.8486796617507935, + "logps/chosen": -510.16583251953125, + "logps/ref_chosen": -269.97369384765625, + "logps/ref_rejected": -235.03164672851562, + "logps/rejected": -572.564208984375, + "loss": 4.7937, + "margin_dpo/margin_mean": 97.34046936035156, + "margin_dpo/margin_std": 192.24783325195312, + "step": 332 + }, + { + "KL/chosen_KL_mean": -243.05345153808594, + "KL/mean": -284.6350402832031, + "KL/rejected_KL_mean": -326.21661376953125, + "KL/std": 145.56326293945312, + "epoch": 0.6973821989528796, + "fcm_dpo/beta": 0.0030364254489541054, + "fcm_dpo/delta": 0.02071891352534294, + "fcm_dpo/margin": 83.16316986083984, + "fcm_dpo/q_t": 0.4418914318084717, + "grad_norm": 94.2875747680664, + "learning_rate": 1.281842711051438e-07, + "logits/chosen": -0.9469788074493408, + "logits/rejected": -0.9081500172615051, + "logps/chosen": -539.8164672851562, + "logps/ref_chosen": -296.76300048828125, + "logps/ref_rejected": -265.97991943359375, + "logps/rejected": -592.1965942382812, + "loss": 4.8583, + "margin_dpo/margin_mean": 83.16317749023438, + "margin_dpo/margin_std": 168.8401641845703, + "step": 333 + }, + { + "KL/chosen_KL_mean": -237.4409942626953, + "KL/mean": -285.7542724609375, + "KL/rejected_KL_mean": -334.0675354003906, + "KL/std": 146.4021453857422, + "epoch": 0.6994764397905759, + "fcm_dpo/beta": 0.0030563974287360907, + "fcm_dpo/delta": -0.020071666687726974, + "fcm_dpo/margin": 96.62651062011719, + "fcm_dpo/q_t": 0.43138545751571655, + "grad_norm": 58.2845573425293, + "learning_rate": 1.2658882646922033e-07, + "logits/chosen": -0.8468398451805115, + "logits/rejected": -0.817967414855957, + "logps/chosen": -538.4777221679688, + "logps/ref_chosen": -301.0367431640625, + "logps/ref_rejected": -268.87652587890625, + "logps/rejected": -602.9440307617188, + "loss": 4.7228, + "margin_dpo/margin_mean": 96.62650299072266, + "margin_dpo/margin_std": 164.7143096923828, + "step": 334 + }, + { + "KL/chosen_KL_mean": -256.3838806152344, + "KL/mean": -304.9598388671875, + "KL/rejected_KL_mean": -353.535888671875, + "KL/std": 136.20851135253906, + "epoch": 0.7015706806282722, + "fcm_dpo/beta": 0.0029724808409810066, + "fcm_dpo/delta": -0.027509108185768127, + "fcm_dpo/margin": 97.15202331542969, + "fcm_dpo/q_t": 0.43334853649139404, + "grad_norm": 62.15879821777344, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.8522113561630249, + "logits/rejected": -0.8463060259819031, + "logps/chosen": -532.5166625976562, + "logps/ref_chosen": -276.13275146484375, + "logps/ref_rejected": -243.44203186035156, + "logps/rejected": -596.9779052734375, + "loss": 4.7403, + "margin_dpo/margin_mean": 97.15201568603516, + "margin_dpo/margin_std": 172.3384246826172, + "step": 335 + }, + { + "KL/chosen_KL_mean": -251.550048828125, + "KL/mean": -300.40789794921875, + "KL/rejected_KL_mean": -349.2657165527344, + "KL/std": 172.0433349609375, + "epoch": 0.7036649214659686, + "fcm_dpo/beta": 0.0029860164504498243, + "fcm_dpo/delta": 0.013530546799302101, + "fcm_dpo/margin": 97.7156753540039, + "fcm_dpo/q_t": 0.4363878071308136, + "grad_norm": 74.54422760009766, + "learning_rate": 1.2341787690142435e-07, + "logits/chosen": -0.8671432733535767, + "logits/rejected": -0.8070932030677795, + "logps/chosen": -497.81268310546875, + "logps/ref_chosen": -246.2626495361328, + "logps/ref_rejected": -261.0617980957031, + "logps/rejected": -610.3275756835938, + "loss": 4.7949, + "margin_dpo/margin_mean": 97.7156753540039, + "margin_dpo/margin_std": 191.96246337890625, + "step": 336 + }, + { + "KL/chosen_KL_mean": -237.27195739746094, + "KL/mean": -291.9794006347656, + "KL/rejected_KL_mean": -346.6868896484375, + "KL/std": 143.3297882080078, + "epoch": 0.7057591623036649, + "fcm_dpo/beta": 0.0029976258520036936, + "fcm_dpo/delta": -0.021170055493712425, + "fcm_dpo/margin": 109.41490936279297, + "fcm_dpo/q_t": 0.42369288206100464, + "grad_norm": 77.50701904296875, + "learning_rate": 1.2184254201795363e-07, + "logits/chosen": -0.8679603338241577, + "logits/rejected": -0.8385904431343079, + "logps/chosen": -504.2657470703125, + "logps/ref_chosen": -266.9937744140625, + "logps/ref_rejected": -253.015625, + "logps/rejected": -599.7025146484375, + "loss": 4.5946, + "margin_dpo/margin_mean": 109.41490936279297, + "margin_dpo/margin_std": 163.00048828125, + "step": 337 + }, + { + "KL/chosen_KL_mean": -235.5108642578125, + "KL/mean": -288.2745056152344, + "KL/rejected_KL_mean": -341.03814697265625, + "KL/std": 152.8268280029297, + "epoch": 0.7078534031413612, + "fcm_dpo/beta": 0.0029197270050644875, + "fcm_dpo/delta": -0.005821624770760536, + "fcm_dpo/margin": 105.52726745605469, + "fcm_dpo/q_t": 0.4291449189186096, + "grad_norm": 75.93712615966797, + "learning_rate": 1.202740798300168e-07, + "logits/chosen": -0.8869338631629944, + "logits/rejected": -0.8708446025848389, + "logps/chosen": -512.1034545898438, + "logps/ref_chosen": -276.5925598144531, + "logps/ref_rejected": -233.979248046875, + "logps/rejected": -575.017333984375, + "loss": 4.6638, + "margin_dpo/margin_mean": 105.52726745605469, + "margin_dpo/margin_std": 172.4578094482422, + "step": 338 + }, + { + "KL/chosen_KL_mean": -237.4051513671875, + "KL/mean": -297.82196044921875, + "KL/rejected_KL_mean": -358.23876953125, + "KL/std": 157.0062255859375, + "epoch": 0.7099476439790576, + "fcm_dpo/beta": 0.0029296469874680042, + "fcm_dpo/delta": -0.021757004782557487, + "fcm_dpo/margin": 120.8335952758789, + "fcm_dpo/q_t": 0.41886672377586365, + "grad_norm": 64.22445678710938, + "learning_rate": 1.1871257444948096e-07, + "logits/chosen": -0.9070875644683838, + "logits/rejected": -0.8940801620483398, + "logps/chosen": -540.9329223632812, + "logps/ref_chosen": -303.5277404785156, + "logps/ref_rejected": -283.11676025390625, + "logps/rejected": -641.35546875, + "loss": 4.5489, + "margin_dpo/margin_mean": 120.83360290527344, + "margin_dpo/margin_std": 184.66961669921875, + "step": 339 + }, + { + "KL/chosen_KL_mean": -246.29815673828125, + "KL/mean": -293.8956604003906, + "KL/rejected_KL_mean": -341.4931640625, + "KL/std": 165.5948486328125, + "epoch": 0.7120418848167539, + "fcm_dpo/beta": 0.0028934727888554335, + "fcm_dpo/delta": 0.01472179964184761, + "fcm_dpo/margin": 95.19498443603516, + "fcm_dpo/q_t": 0.43556487560272217, + "grad_norm": 89.77741241455078, + "learning_rate": 1.1715810961514072e-07, + "logits/chosen": -0.8359464406967163, + "logits/rejected": -0.8298177719116211, + "logps/chosen": -507.82391357421875, + "logps/ref_chosen": -261.5257568359375, + "logps/ref_rejected": -259.39862060546875, + "logps/rejected": -600.8917236328125, + "loss": 4.88, + "margin_dpo/margin_mean": 95.19498443603516, + "margin_dpo/margin_std": 201.7469482421875, + "step": 340 + }, + { + "KL/chosen_KL_mean": -288.7161865234375, + "KL/mean": -333.5431213378906, + "KL/rejected_KL_mean": -378.3700256347656, + "KL/std": 184.27386474609375, + "epoch": 0.7141361256544503, + "fcm_dpo/beta": 0.002914420096203685, + "fcm_dpo/delta": -0.00907097663730383, + "fcm_dpo/margin": 89.65385437011719, + "fcm_dpo/q_t": 0.4414941966533661, + "grad_norm": 75.35196685791016, + "learning_rate": 1.1561076868822755e-07, + "logits/chosen": -0.8593753576278687, + "logits/rejected": -0.8326888680458069, + "logps/chosen": -604.6196899414062, + "logps/ref_chosen": -315.903564453125, + "logps/ref_rejected": -308.02392578125, + "logps/rejected": -686.3939819335938, + "loss": 4.9406, + "margin_dpo/margin_mean": 89.65385437011719, + "margin_dpo/margin_std": 208.85565185546875, + "step": 341 + }, + { + "KL/chosen_KL_mean": -238.28636169433594, + "KL/mean": -286.4154052734375, + "KL/rejected_KL_mean": -334.54443359375, + "KL/std": 143.09034729003906, + "epoch": 0.7162303664921466, + "fcm_dpo/beta": 0.002888055983930826, + "fcm_dpo/delta": -0.005859066266566515, + "fcm_dpo/margin": 96.25808715820312, + "fcm_dpo/q_t": 0.4352652430534363, + "grad_norm": 82.94450378417969, + "learning_rate": 1.1407063464793965e-07, + "logits/chosen": -0.8524309992790222, + "logits/rejected": -0.8473560810089111, + "logps/chosen": -507.4649963378906, + "logps/ref_chosen": -269.17864990234375, + "logps/ref_rejected": -260.8977355957031, + "logps/rejected": -595.442138671875, + "loss": 4.7214, + "margin_dpo/margin_mean": 96.25808715820312, + "margin_dpo/margin_std": 158.40966796875, + "step": 342 + }, + { + "KL/chosen_KL_mean": -251.1302032470703, + "KL/mean": -289.5614318847656, + "KL/rejected_KL_mean": -327.99267578125, + "KL/std": 144.47100830078125, + "epoch": 0.7183246073298429, + "fcm_dpo/beta": 0.0028825635090470314, + "fcm_dpo/delta": 0.007316844072192907, + "fcm_dpo/margin": 76.86248779296875, + "fcm_dpo/q_t": 0.4476861357688904, + "grad_norm": 78.4572525024414, + "learning_rate": 1.125377900869913e-07, + "logits/chosen": -0.8445292711257935, + "logits/rejected": -0.8152074813842773, + "logps/chosen": -561.8502197265625, + "logps/ref_chosen": -310.719970703125, + "logps/ref_rejected": -263.5224914550781, + "logps/rejected": -591.51513671875, + "loss": 4.9124, + "margin_dpo/margin_mean": 76.86248779296875, + "margin_dpo/margin_std": 157.17347717285156, + "step": 343 + }, + { + "KL/chosen_KL_mean": -247.63592529296875, + "KL/mean": -293.60247802734375, + "KL/rejected_KL_mean": -339.5689697265625, + "KL/std": 140.92300415039062, + "epoch": 0.7204188481675393, + "fcm_dpo/beta": 0.0028950828127563, + "fcm_dpo/delta": -0.003131479024887085, + "fcm_dpo/margin": 91.93303680419922, + "fcm_dpo/q_t": 0.4373406767845154, + "grad_norm": 72.97909545898438, + "learning_rate": 1.110123172071844e-07, + "logits/chosen": -0.8511172533035278, + "logits/rejected": -0.8328704237937927, + "logps/chosen": -549.4358520507812, + "logps/ref_chosen": -301.7999267578125, + "logps/ref_rejected": -257.9061584472656, + "logps/rejected": -597.4751586914062, + "loss": 4.7968, + "margin_dpo/margin_mean": 91.93302917480469, + "margin_dpo/margin_std": 167.35589599609375, + "step": 344 + }, + { + "KL/chosen_KL_mean": -232.2271728515625, + "KL/mean": -273.35272216796875, + "KL/rejected_KL_mean": -314.4782409667969, + "KL/std": 139.15582275390625, + "epoch": 0.7225130890052356, + "fcm_dpo/beta": 0.0028755811508744955, + "fcm_dpo/delta": -0.004357065074145794, + "fcm_dpo/margin": 82.25109100341797, + "fcm_dpo/q_t": 0.444185733795166, + "grad_norm": 76.50686645507812, + "learning_rate": 1.09494297815e-07, + "logits/chosen": -0.8535346984863281, + "logits/rejected": -0.8539408445358276, + "logps/chosen": -515.24560546875, + "logps/ref_chosen": -283.0184326171875, + "logps/ref_rejected": -266.8457336425781, + "logps/rejected": -581.323974609375, + "loss": 4.8509, + "margin_dpo/margin_mean": 82.25108337402344, + "margin_dpo/margin_std": 142.70669555664062, + "step": 345 + }, + { + "KL/chosen_KL_mean": -243.66827392578125, + "KL/mean": -296.9436950683594, + "KL/rejected_KL_mean": -350.21917724609375, + "KL/std": 160.29945373535156, + "epoch": 0.724607329842932, + "fcm_dpo/beta": 0.0028548564296215773, + "fcm_dpo/delta": -0.013966667465865612, + "fcm_dpo/margin": 106.5509033203125, + "fcm_dpo/q_t": 0.4294491708278656, + "grad_norm": 50.279972076416016, + "learning_rate": 1.0798381331721107e-07, + "logits/chosen": -0.9294825792312622, + "logits/rejected": -0.87758469581604, + "logps/chosen": -512.1094970703125, + "logps/ref_chosen": -268.44122314453125, + "logps/ref_rejected": -227.8225860595703, + "logps/rejected": -578.041748046875, + "loss": 4.666, + "margin_dpo/margin_mean": 106.5509033203125, + "margin_dpo/margin_std": 163.5757293701172, + "step": 346 + }, + { + "KL/chosen_KL_mean": -224.2758331298828, + "KL/mean": -267.6025390625, + "KL/rejected_KL_mean": -310.92919921875, + "KL/std": 138.48739624023438, + "epoch": 0.7267015706806282, + "fcm_dpo/beta": 0.002854297636076808, + "fcm_dpo/delta": 0.007548038847744465, + "fcm_dpo/margin": 86.65332794189453, + "fcm_dpo/q_t": 0.4415615499019623, + "grad_norm": 83.579833984375, + "learning_rate": 1.0648094471651722e-07, + "logits/chosen": -0.7760555148124695, + "logits/rejected": -0.8024675250053406, + "logps/chosen": -497.9794006347656, + "logps/ref_chosen": -273.70355224609375, + "logps/ref_rejected": -243.65521240234375, + "logps/rejected": -554.5843505859375, + "loss": 4.8312, + "margin_dpo/margin_mean": 86.65332794189453, + "margin_dpo/margin_std": 157.4734344482422, + "step": 347 + }, + { + "KL/chosen_KL_mean": -220.09567260742188, + "KL/mean": -258.8450622558594, + "KL/rejected_KL_mean": -297.594482421875, + "KL/std": 147.97817993164062, + "epoch": 0.7287958115183246, + "fcm_dpo/beta": 0.0028738253749907017, + "fcm_dpo/delta": 0.0110637117177248, + "fcm_dpo/margin": 77.49883270263672, + "fcm_dpo/q_t": 0.4496265649795532, + "grad_norm": 75.31076049804688, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": -0.892849326133728, + "logits/rejected": -0.8771178722381592, + "logps/chosen": -505.7370910644531, + "logps/ref_chosen": -285.64141845703125, + "logps/ref_rejected": -265.6270446777344, + "logps/rejected": -563.2215576171875, + "loss": 4.9376, + "margin_dpo/margin_mean": 77.49882507324219, + "margin_dpo/margin_std": 166.94793701171875, + "step": 348 + }, + { + "KL/chosen_KL_mean": -220.64576721191406, + "KL/mean": -279.0647277832031, + "KL/rejected_KL_mean": -337.48370361328125, + "KL/std": 160.11648559570312, + "epoch": 0.7308900523560209, + "fcm_dpo/beta": 0.0028767730109393597, + "fcm_dpo/delta": -0.013212434016168118, + "fcm_dpo/margin": 116.83795928955078, + "fcm_dpo/q_t": 0.42093393206596375, + "grad_norm": 66.55834197998047, + "learning_rate": 1.0349837717080347e-07, + "logits/chosen": -0.8378231525421143, + "logits/rejected": -0.8238610625267029, + "logps/chosen": -548.9632568359375, + "logps/ref_chosen": -328.3175048828125, + "logps/ref_rejected": -292.37872314453125, + "logps/rejected": -629.8624267578125, + "loss": 4.6037, + "margin_dpo/margin_mean": 116.83795928955078, + "margin_dpo/margin_std": 184.72499084472656, + "step": 349 + }, + { + "KL/chosen_KL_mean": -237.5376434326172, + "KL/mean": -285.01214599609375, + "KL/rejected_KL_mean": -332.4866027832031, + "KL/std": 148.0513916015625, + "epoch": 0.7329842931937173, + "fcm_dpo/beta": 0.0028601842932403088, + "fcm_dpo/delta": 0.004661480896174908, + "fcm_dpo/margin": 94.94898986816406, + "fcm_dpo/q_t": 0.43736475706100464, + "grad_norm": 80.1038818359375, + "learning_rate": 1.0201883817182949e-07, + "logits/chosen": -0.8289676904678345, + "logits/rejected": -0.8410451412200928, + "logps/chosen": -530.34228515625, + "logps/ref_chosen": -292.8046569824219, + "logps/ref_rejected": -250.35504150390625, + "logps/rejected": -582.8416748046875, + "loss": 4.7774, + "margin_dpo/margin_mean": 94.94898986816406, + "margin_dpo/margin_std": 173.54653930664062, + "step": 350 + }, + { + "KL/chosen_KL_mean": -223.345458984375, + "KL/mean": -254.21771240234375, + "KL/rejected_KL_mean": -285.0899658203125, + "KL/std": 138.79812622070312, + "epoch": 0.7350785340314137, + "fcm_dpo/beta": 0.0028854927513748407, + "fcm_dpo/delta": 0.013731059618294239, + "fcm_dpo/margin": 61.744510650634766, + "fcm_dpo/q_t": 0.4585198163986206, + "grad_norm": 70.86201477050781, + "learning_rate": 1.0054723495346482e-07, + "logits/chosen": -0.8867701292037964, + "logits/rejected": -0.8659510612487793, + "logps/chosen": -535.2344970703125, + "logps/ref_chosen": -311.8890380859375, + "logps/ref_rejected": -263.59033203125, + "logps/rejected": -548.6802978515625, + "loss": 5.113, + "margin_dpo/margin_mean": 61.744510650634766, + "margin_dpo/margin_std": 169.5723876953125, + "step": 351 + }, + { + "KL/chosen_KL_mean": -210.1036376953125, + "KL/mean": -268.3479309082031, + "KL/rejected_KL_mean": -326.5922546386719, + "KL/std": 149.0188446044922, + "epoch": 0.7371727748691099, + "fcm_dpo/beta": 0.0028885467909276485, + "fcm_dpo/delta": -0.00935527216643095, + "fcm_dpo/margin": 116.48863220214844, + "fcm_dpo/q_t": 0.42103269696235657, + "grad_norm": 57.36745834350586, + "learning_rate": 9.908364643332398e-08, + "logits/chosen": -0.8118300437927246, + "logits/rejected": -0.7807732820510864, + "logps/chosen": -465.0115051269531, + "logps/ref_chosen": -254.9078826904297, + "logps/ref_rejected": -257.1688232421875, + "logps/rejected": -583.7610473632812, + "loss": 4.5527, + "margin_dpo/margin_mean": 116.48863983154297, + "margin_dpo/margin_std": 165.00392150878906, + "step": 352 + }, + { + "KL/chosen_KL_mean": -229.11819458007812, + "KL/mean": -279.8221740722656, + "KL/rejected_KL_mean": -330.526123046875, + "KL/std": 147.36636352539062, + "epoch": 0.7392670157068063, + "fcm_dpo/beta": 0.0028685785364359617, + "fcm_dpo/delta": -0.013993466272950172, + "fcm_dpo/margin": 101.40792846679688, + "fcm_dpo/q_t": 0.4323953688144684, + "grad_norm": 93.38015747070312, + "learning_rate": 9.76281510992176e-08, + "logits/chosen": -0.8352831602096558, + "logits/rejected": -0.8298249840736389, + "logps/chosen": -499.4942626953125, + "logps/ref_chosen": -270.3760681152344, + "logps/ref_rejected": -264.65234375, + "logps/rejected": -595.178466796875, + "loss": 4.7011, + "margin_dpo/margin_mean": 101.40792846679688, + "margin_dpo/margin_std": 164.58172607421875, + "step": 353 + }, + { + "KL/chosen_KL_mean": -226.212646484375, + "KL/mean": -265.5102233886719, + "KL/rejected_KL_mean": -304.8078308105469, + "KL/std": 154.871826171875, + "epoch": 0.7413612565445026, + "fcm_dpo/beta": 0.002862285589799285, + "fcm_dpo/delta": 0.012168960645794868, + "fcm_dpo/margin": 78.59518432617188, + "fcm_dpo/q_t": 0.4484597146511078, + "grad_norm": 77.23882293701172, + "learning_rate": 9.618082700494318e-08, + "logits/chosen": -0.8314879536628723, + "logits/rejected": -0.8639715909957886, + "logps/chosen": -483.8612365722656, + "logps/ref_chosen": -257.6485595703125, + "logps/ref_rejected": -246.94203186035156, + "logps/rejected": -551.7498779296875, + "loss": 4.9439, + "margin_dpo/margin_mean": 78.59518432617188, + "margin_dpo/margin_std": 171.47744750976562, + "step": 354 + }, + { + "KL/chosen_KL_mean": -228.62823486328125, + "KL/mean": -280.390625, + "KL/rejected_KL_mean": -332.1530456542969, + "KL/std": 161.08724975585938, + "epoch": 0.743455497382199, + "fcm_dpo/beta": 0.0028833611868321896, + "fcm_dpo/delta": 0.009556437842547894, + "fcm_dpo/margin": 103.52484893798828, + "fcm_dpo/q_t": 0.431077778339386, + "grad_norm": 55.02450942993164, + "learning_rate": 9.474175176609956e-08, + "logits/chosen": -0.8792758584022522, + "logits/rejected": -0.8736676573753357, + "logps/chosen": -521.9815673828125, + "logps/ref_chosen": -293.35333251953125, + "logps/ref_rejected": -275.6051940917969, + "logps/rejected": -607.7582397460938, + "loss": 4.7879, + "margin_dpo/margin_mean": 103.52484893798828, + "margin_dpo/margin_std": 198.6239013671875, + "step": 355 + }, + { + "KL/chosen_KL_mean": -216.16531372070312, + "KL/mean": -253.62515258789062, + "KL/rejected_KL_mean": -291.08502197265625, + "KL/std": 134.0445556640625, + "epoch": 0.7455497382198953, + "fcm_dpo/beta": 0.0028942120261490345, + "fcm_dpo/delta": -0.0024351300671696663, + "fcm_dpo/margin": 74.91969299316406, + "fcm_dpo/q_t": 0.44854995608329773, + "grad_norm": 71.81739044189453, + "learning_rate": 9.331100255592436e-08, + "logits/chosen": -0.7871492505073547, + "logits/rejected": -0.8126851320266724, + "logps/chosen": -420.4208068847656, + "logps/ref_chosen": -204.25550842285156, + "logps/ref_rejected": -213.467529296875, + "logps/rejected": -504.552490234375, + "loss": 4.9034, + "margin_dpo/margin_mean": 74.91969299316406, + "margin_dpo/margin_std": 139.6249542236328, + "step": 356 + }, + { + "KL/chosen_KL_mean": -239.94329833984375, + "KL/mean": -289.7812805175781, + "KL/rejected_KL_mean": -339.6192932128906, + "KL/std": 154.73379516601562, + "epoch": 0.7476439790575916, + "fcm_dpo/beta": 0.00291292741894722, + "fcm_dpo/delta": 0.021629903465509415, + "fcm_dpo/margin": 99.67597961425781, + "fcm_dpo/q_t": 0.433463990688324, + "grad_norm": 69.52693939208984, + "learning_rate": 9.18886561011557e-08, + "logits/chosen": -0.7813151478767395, + "logits/rejected": -0.7780415415763855, + "logps/chosen": -506.3138427734375, + "logps/ref_chosen": -266.3705749511719, + "logps/ref_rejected": -239.04490661621094, + "logps/rejected": -578.6642456054688, + "loss": 4.7521, + "margin_dpo/margin_mean": 99.67597961425781, + "margin_dpo/margin_std": 178.00946044921875, + "step": 357 + }, + { + "KL/chosen_KL_mean": -217.33197021484375, + "KL/mean": -272.29388427734375, + "KL/rejected_KL_mean": -327.2558288574219, + "KL/std": 141.83389282226562, + "epoch": 0.749738219895288, + "fcm_dpo/beta": 0.00295689282938838, + "fcm_dpo/delta": 0.014568077400326729, + "fcm_dpo/margin": 109.92384338378906, + "fcm_dpo/q_t": 0.42453300952911377, + "grad_norm": 83.9714126586914, + "learning_rate": 9.047478867791731e-08, + "logits/chosen": -0.8699600696563721, + "logits/rejected": -0.8453044891357422, + "logps/chosen": -516.4794311523438, + "logps/ref_chosen": -299.1474609375, + "logps/ref_rejected": -257.2531433105469, + "logps/rejected": -584.509033203125, + "loss": 4.6161, + "margin_dpo/margin_mean": 109.92383575439453, + "margin_dpo/margin_std": 175.24615478515625, + "step": 358 + }, + { + "KL/chosen_KL_mean": -230.4601593017578, + "KL/mean": -278.14129638671875, + "KL/rejected_KL_mean": -325.82244873046875, + "KL/std": 147.65829467773438, + "epoch": 0.7518324607329843, + "fcm_dpo/beta": 0.0029695353005081415, + "fcm_dpo/delta": 0.0013146661221981049, + "fcm_dpo/margin": 95.36227416992188, + "fcm_dpo/q_t": 0.43455877900123596, + "grad_norm": 70.06732177734375, + "learning_rate": 8.906947610762825e-08, + "logits/chosen": -0.8403257131576538, + "logits/rejected": -0.8475351333618164, + "logps/chosen": -533.4580078125, + "logps/ref_chosen": -302.99786376953125, + "logps/ref_rejected": -260.4137268066406, + "logps/rejected": -586.2362060546875, + "loss": 4.7681, + "margin_dpo/margin_mean": 95.36227416992188, + "margin_dpo/margin_std": 175.98709106445312, + "step": 359 + }, + { + "KL/chosen_KL_mean": -232.94993591308594, + "KL/mean": -278.8365783691406, + "KL/rejected_KL_mean": -324.7232360839844, + "KL/std": 160.8883056640625, + "epoch": 0.7539267015706806, + "fcm_dpo/beta": 0.0029307494405657053, + "fcm_dpo/delta": -0.03134762495756149, + "fcm_dpo/margin": 91.77330780029297, + "fcm_dpo/q_t": 0.43876707553863525, + "grad_norm": 60.748844146728516, + "learning_rate": 8.76727937529367e-08, + "logits/chosen": -0.8615961074829102, + "logits/rejected": -0.8414865136146545, + "logps/chosen": -542.5614013671875, + "logps/ref_chosen": -309.6114501953125, + "logps/ref_rejected": -256.64031982421875, + "logps/rejected": -581.363525390625, + "loss": 4.8976, + "margin_dpo/margin_mean": 91.77330780029297, + "margin_dpo/margin_std": 189.3778076171875, + "step": 360 + }, + { + "KL/chosen_KL_mean": -196.89012145996094, + "KL/mean": -256.0363464355469, + "KL/rejected_KL_mean": -315.18255615234375, + "KL/std": 154.51834106445312, + "epoch": 0.7560209424083769, + "fcm_dpo/beta": 0.0029320104513317347, + "fcm_dpo/delta": 0.015726841986179352, + "fcm_dpo/margin": 118.29248046875, + "fcm_dpo/q_t": 0.421514093875885, + "grad_norm": 64.89899444580078, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": -0.803794264793396, + "logits/rejected": -0.7853226065635681, + "logps/chosen": -460.2698669433594, + "logps/ref_chosen": -263.3797607421875, + "logps/ref_rejected": -271.18157958984375, + "logps/rejected": -586.3641357421875, + "loss": 4.5926, + "margin_dpo/margin_mean": 118.29248046875, + "margin_dpo/margin_std": 193.5961456298828, + "step": 361 + }, + { + "KL/chosen_KL_mean": -219.24996948242188, + "KL/mean": -266.0592956542969, + "KL/rejected_KL_mean": -312.86865234375, + "KL/std": 129.30587768554688, + "epoch": 0.7581151832460733, + "fcm_dpo/beta": 0.002948556561022997, + "fcm_dpo/delta": 0.006594155449420214, + "fcm_dpo/margin": 93.61869812011719, + "fcm_dpo/q_t": 0.43606704473495483, + "grad_norm": 72.42217254638672, + "learning_rate": 8.490561882286135e-08, + "logits/chosen": -0.8361051082611084, + "logits/rejected": -0.81890469789505, + "logps/chosen": -522.5083618164062, + "logps/ref_chosen": -303.2583923339844, + "logps/ref_rejected": -243.22891235351562, + "logps/rejected": -556.0975341796875, + "loss": 4.7161, + "margin_dpo/margin_mean": 93.61870574951172, + "margin_dpo/margin_std": 146.4253387451172, + "step": 362 + }, + { + "KL/chosen_KL_mean": -224.68502807617188, + "KL/mean": -277.4530029296875, + "KL/rejected_KL_mean": -330.220947265625, + "KL/std": 156.96578979492188, + "epoch": 0.7602094240837697, + "fcm_dpo/beta": 0.002929537557065487, + "fcm_dpo/delta": -0.03551473096013069, + "fcm_dpo/margin": 105.53596496582031, + "fcm_dpo/q_t": 0.42976054549217224, + "grad_norm": 71.42375183105469, + "learning_rate": 8.353527464267104e-08, + "logits/chosen": -0.8412438035011292, + "logits/rejected": -0.7956109642982483, + "logps/chosen": -528.0322875976562, + "logps/ref_chosen": -303.34722900390625, + "logps/ref_rejected": -262.05419921875, + "logps/rejected": -592.2752075195312, + "loss": 4.7065, + "margin_dpo/margin_mean": 105.53596496582031, + "margin_dpo/margin_std": 184.98667907714844, + "step": 363 + }, + { + "KL/chosen_KL_mean": -254.75527954101562, + "KL/mean": -303.8872375488281, + "KL/rejected_KL_mean": -353.0191650390625, + "KL/std": 167.35247802734375, + "epoch": 0.762303664921466, + "fcm_dpo/beta": 0.002902527106925845, + "fcm_dpo/delta": 0.02750171534717083, + "fcm_dpo/margin": 98.26390838623047, + "fcm_dpo/q_t": 0.43715596199035645, + "grad_norm": 70.5536117553711, + "learning_rate": 8.217385746050742e-08, + "logits/chosen": -0.8307126760482788, + "logits/rejected": -0.8298197984695435, + "logps/chosen": -540.2990112304688, + "logps/ref_chosen": -285.54376220703125, + "logps/ref_rejected": -284.84619140625, + "logps/rejected": -637.8654174804688, + "loss": 4.8201, + "margin_dpo/margin_mean": 98.263916015625, + "margin_dpo/margin_std": 200.9420623779297, + "step": 364 + }, + { + "KL/chosen_KL_mean": -238.25563049316406, + "KL/mean": -282.6103515625, + "KL/rejected_KL_mean": -326.96502685546875, + "KL/std": 162.19873046875, + "epoch": 0.7643979057591623, + "fcm_dpo/beta": 0.002954564755782485, + "fcm_dpo/delta": 0.009645760990679264, + "fcm_dpo/margin": 88.70938110351562, + "fcm_dpo/q_t": 0.44021692872047424, + "grad_norm": 70.69696807861328, + "learning_rate": 8.082144028504231e-08, + "logits/chosen": -0.8417816758155823, + "logits/rejected": -0.8418366312980652, + "logps/chosen": -513.04345703125, + "logps/ref_chosen": -274.7878112792969, + "logps/ref_rejected": -256.5738220214844, + "logps/rejected": -583.5388793945312, + "loss": 4.8457, + "margin_dpo/margin_mean": 88.70938110351562, + "margin_dpo/margin_std": 176.14007568359375, + "step": 365 + }, + { + "KL/chosen_KL_mean": -222.59596252441406, + "KL/mean": -271.5234680175781, + "KL/rejected_KL_mean": -320.450927734375, + "KL/std": 141.25015258789062, + "epoch": 0.7664921465968586, + "fcm_dpo/beta": 0.0030219045002013445, + "fcm_dpo/delta": 0.037596531212329865, + "fcm_dpo/margin": 97.85496520996094, + "fcm_dpo/q_t": 0.4315846860408783, + "grad_norm": 70.44728088378906, + "learning_rate": 7.947809564230445e-08, + "logits/chosen": -0.7966880798339844, + "logits/rejected": -0.8008946776390076, + "logps/chosen": -509.24560546875, + "logps/ref_chosen": -286.6496276855469, + "logps/ref_rejected": -251.97140502929688, + "logps/rejected": -572.42236328125, + "loss": 4.717, + "margin_dpo/margin_mean": 97.85496520996094, + "margin_dpo/margin_std": 171.53121948242188, + "step": 366 + }, + { + "KL/chosen_KL_mean": -226.7736358642578, + "KL/mean": -278.9706115722656, + "KL/rejected_KL_mean": -331.16766357421875, + "KL/std": 146.42007446289062, + "epoch": 0.768586387434555, + "fcm_dpo/beta": 0.00305999256670475, + "fcm_dpo/delta": 0.010715863667428493, + "fcm_dpo/margin": 104.39400482177734, + "fcm_dpo/q_t": 0.4274834990501404, + "grad_norm": 82.6015625, + "learning_rate": 7.814389557179016e-08, + "logits/chosen": -0.8351811766624451, + "logits/rejected": -0.8077444434165955, + "logps/chosen": -528.7186279296875, + "logps/ref_chosen": -301.9449768066406, + "logps/ref_rejected": -265.5677185058594, + "logps/rejected": -596.7353515625, + "loss": 4.6422, + "margin_dpo/margin_mean": 104.39400482177734, + "margin_dpo/margin_std": 168.87966918945312, + "step": 367 + }, + { + "KL/chosen_KL_mean": -209.43209838867188, + "KL/mean": -276.49969482421875, + "KL/rejected_KL_mean": -343.56732177734375, + "KL/std": 147.89605712890625, + "epoch": 0.7706806282722513, + "fcm_dpo/beta": 0.0030316186603158712, + "fcm_dpo/delta": -0.04178418964147568, + "fcm_dpo/margin": 134.13523864746094, + "fcm_dpo/q_t": 0.405577689409256, + "grad_norm": 76.57606506347656, + "learning_rate": 7.681891162260015e-08, + "logits/chosen": -0.7988978028297424, + "logits/rejected": -0.8013542890548706, + "logps/chosen": -504.05865478515625, + "logps/ref_chosen": -294.62652587890625, + "logps/ref_rejected": -258.7628479003906, + "logps/rejected": -602.3301391601562, + "loss": 4.3154, + "margin_dpo/margin_mean": 134.13523864746094, + "margin_dpo/margin_std": 154.99842834472656, + "step": 368 + }, + { + "KL/chosen_KL_mean": -225.83438110351562, + "KL/mean": -273.8876647949219, + "KL/rejected_KL_mean": -321.9409484863281, + "KL/std": 137.2335205078125, + "epoch": 0.7727748691099476, + "fcm_dpo/beta": 0.0030030980706214905, + "fcm_dpo/delta": 0.01607823558151722, + "fcm_dpo/margin": 96.10658264160156, + "fcm_dpo/q_t": 0.4320639371871948, + "grad_norm": 82.7309341430664, + "learning_rate": 7.550321484960251e-08, + "logits/chosen": -0.8810120820999146, + "logits/rejected": -0.8641123175621033, + "logps/chosen": -508.340087890625, + "logps/ref_chosen": -282.5057373046875, + "logps/ref_rejected": -266.41607666015625, + "logps/rejected": -588.3570556640625, + "loss": 4.6647, + "margin_dpo/margin_mean": 96.10657501220703, + "margin_dpo/margin_std": 144.0449981689453, + "step": 369 + }, + { + "KL/chosen_KL_mean": -215.18609619140625, + "KL/mean": -274.7394104003906, + "KL/rejected_KL_mean": -334.2928161621094, + "KL/std": 153.53570556640625, + "epoch": 0.774869109947644, + "fcm_dpo/beta": 0.003025064943358302, + "fcm_dpo/delta": -0.0023751449771225452, + "fcm_dpo/margin": 119.10672760009766, + "fcm_dpo/q_t": 0.4178846776485443, + "grad_norm": 62.49055480957031, + "learning_rate": 7.419687580962222e-08, + "logits/chosen": -0.8580411076545715, + "logits/rejected": -0.8811969757080078, + "logps/chosen": -466.1924743652344, + "logps/ref_chosen": -251.00640869140625, + "logps/ref_rejected": -238.12542724609375, + "logps/rejected": -572.418212890625, + "loss": 4.5629, + "margin_dpo/margin_mean": 119.10673522949219, + "margin_dpo/margin_std": 193.12779235839844, + "step": 370 + }, + { + "KL/chosen_KL_mean": -250.4105682373047, + "KL/mean": -285.2479248046875, + "KL/rejected_KL_mean": -320.0852355957031, + "KL/std": 149.4564666748047, + "epoch": 0.7769633507853403, + "fcm_dpo/beta": 0.003027756232768297, + "fcm_dpo/delta": -0.0017022611573338509, + "fcm_dpo/margin": 69.6746597290039, + "fcm_dpo/q_t": 0.450680136680603, + "grad_norm": 105.20964050292969, + "learning_rate": 7.289996455765748e-08, + "logits/chosen": -0.8298521041870117, + "logits/rejected": -0.8093293309211731, + "logps/chosen": -547.0697021484375, + "logps/ref_chosen": -296.6591491699219, + "logps/ref_rejected": -251.14675903320312, + "logps/rejected": -571.23193359375, + "loss": 4.9843, + "margin_dpo/margin_mean": 69.67466735839844, + "margin_dpo/margin_std": 154.35647583007812, + "step": 371 + }, + { + "KL/chosen_KL_mean": -240.75746154785156, + "KL/mean": -299.1548156738281, + "KL/rejected_KL_mean": -357.55224609375, + "KL/std": 165.2203826904297, + "epoch": 0.7790575916230367, + "fcm_dpo/beta": 0.0029513647314161062, + "fcm_dpo/delta": -0.024020541459321976, + "fcm_dpo/margin": 116.7947998046875, + "fcm_dpo/q_t": 0.4221827983856201, + "grad_norm": 70.41334533691406, + "learning_rate": 7.161255064312283e-08, + "logits/chosen": -0.7842466235160828, + "logits/rejected": -0.7680008411407471, + "logps/chosen": -572.12890625, + "logps/ref_chosen": -331.3714599609375, + "logps/ref_rejected": -285.56805419921875, + "logps/rejected": -643.1203002929688, + "loss": 4.6121, + "margin_dpo/margin_mean": 116.7947998046875, + "margin_dpo/margin_std": 187.99862670898438, + "step": 372 + }, + { + "KL/chosen_KL_mean": -226.25857543945312, + "KL/mean": -281.29608154296875, + "KL/rejected_KL_mean": -336.3335876464844, + "KL/std": 142.73548889160156, + "epoch": 0.7811518324607329, + "fcm_dpo/beta": 0.002992741297930479, + "fcm_dpo/delta": 0.03157367929816246, + "fcm_dpo/margin": 110.07496643066406, + "fcm_dpo/q_t": 0.42270153760910034, + "grad_norm": 86.4478988647461, + "learning_rate": 7.033470310611945e-08, + "logits/chosen": -0.8854488730430603, + "logits/rejected": -0.8503902554512024, + "logps/chosen": -548.2015380859375, + "logps/ref_chosen": -321.9429931640625, + "logps/ref_rejected": -271.2288513183594, + "logps/rejected": -607.5624389648438, + "loss": 4.5869, + "margin_dpo/margin_mean": 110.0749740600586, + "margin_dpo/margin_std": 165.2940216064453, + "step": 373 + }, + { + "KL/chosen_KL_mean": -236.50103759765625, + "KL/mean": -282.9904479980469, + "KL/rejected_KL_mean": -329.4798278808594, + "KL/std": 160.59323120117188, + "epoch": 0.7832460732984293, + "fcm_dpo/beta": 0.0029890371952205896, + "fcm_dpo/delta": -0.015482398681342602, + "fcm_dpo/margin": 92.97879028320312, + "fcm_dpo/q_t": 0.43591830134391785, + "grad_norm": 74.21908569335938, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": -0.8732917904853821, + "logits/rejected": -0.8661702871322632, + "logps/chosen": -555.6695556640625, + "logps/ref_chosen": -319.1685485839844, + "logps/ref_rejected": -284.6263732910156, + "logps/rejected": -614.1061401367188, + "loss": 4.773, + "margin_dpo/margin_mean": 92.97879028320312, + "margin_dpo/margin_std": 169.21636962890625, + "step": 374 + }, + { + "KL/chosen_KL_mean": -245.4715576171875, + "KL/mean": -289.0046691894531, + "KL/rejected_KL_mean": -332.53778076171875, + "KL/std": 133.46920776367188, + "epoch": 0.7853403141361257, + "fcm_dpo/beta": 0.0029952474869787693, + "fcm_dpo/delta": -0.0004676980897784233, + "fcm_dpo/margin": 87.06620788574219, + "fcm_dpo/q_t": 0.43925830721855164, + "grad_norm": 74.76742553710938, + "learning_rate": 6.780798075635675e-08, + "logits/chosen": -0.8679319620132446, + "logits/rejected": -0.8446568250656128, + "logps/chosen": -560.3473510742188, + "logps/ref_chosen": -314.87579345703125, + "logps/ref_rejected": -259.1965026855469, + "logps/rejected": -591.7342529296875, + "loss": 4.8, + "margin_dpo/margin_mean": 87.06620788574219, + "margin_dpo/margin_std": 155.70870971679688, + "step": 375 + }, + { + "KL/chosen_KL_mean": -241.67770385742188, + "KL/mean": -294.02313232421875, + "KL/rejected_KL_mean": -346.36859130859375, + "KL/std": 157.73382568359375, + "epoch": 0.787434554973822, + "fcm_dpo/beta": 0.0029938980005681515, + "fcm_dpo/delta": -0.005739666521549225, + "fcm_dpo/margin": 104.69088745117188, + "fcm_dpo/q_t": 0.42898622155189514, + "grad_norm": 100.38687133789062, + "learning_rate": 6.655924144404906e-08, + "logits/chosen": -0.8551640510559082, + "logits/rejected": -0.8558748364448547, + "logps/chosen": -529.3509521484375, + "logps/ref_chosen": -287.6732482910156, + "logps/ref_rejected": -256.6697082519531, + "logps/rejected": -603.0382690429688, + "loss": 4.7103, + "margin_dpo/margin_mean": 104.69087982177734, + "margin_dpo/margin_std": 189.6014404296875, + "step": 376 + }, + { + "KL/chosen_KL_mean": -258.047119140625, + "KL/mean": -295.9388732910156, + "KL/rejected_KL_mean": -333.83062744140625, + "KL/std": 157.74354553222656, + "epoch": 0.7895287958115184, + "fcm_dpo/beta": 0.00297717796638608, + "fcm_dpo/delta": 0.003270770888775587, + "fcm_dpo/margin": 75.78350830078125, + "fcm_dpo/q_t": 0.4478086233139038, + "grad_norm": 64.10608673095703, + "learning_rate": 6.532033950290885e-08, + "logits/chosen": -0.8300923109054565, + "logits/rejected": -0.8211076855659485, + "logps/chosen": -563.30859375, + "logps/ref_chosen": -305.261474609375, + "logps/ref_rejected": -271.8887023925781, + "logps/rejected": -605.7192993164062, + "loss": 4.9974, + "margin_dpo/margin_mean": 75.78350830078125, + "margin_dpo/margin_std": 182.6964874267578, + "step": 377 + }, + { + "KL/chosen_KL_mean": -248.6783905029297, + "KL/mean": -296.946533203125, + "KL/rejected_KL_mean": -345.21466064453125, + "KL/std": 163.25067138671875, + "epoch": 0.7916230366492146, + "fcm_dpo/beta": 0.0030068312771618366, + "fcm_dpo/delta": 0.02455754391849041, + "fcm_dpo/margin": 96.5362548828125, + "fcm_dpo/q_t": 0.4341698884963989, + "grad_norm": 74.27600860595703, + "learning_rate": 6.409134137148736e-08, + "logits/chosen": -0.8239084482192993, + "logits/rejected": -0.8121182322502136, + "logps/chosen": -530.2080078125, + "logps/ref_chosen": -281.5295715332031, + "logps/ref_rejected": -296.980224609375, + "logps/rejected": -642.1949462890625, + "loss": 4.7595, + "margin_dpo/margin_mean": 96.5362548828125, + "margin_dpo/margin_std": 179.10707092285156, + "step": 378 + }, + { + "KL/chosen_KL_mean": -228.6240234375, + "KL/mean": -274.6946105957031, + "KL/rejected_KL_mean": -320.76519775390625, + "KL/std": 132.6265869140625, + "epoch": 0.793717277486911, + "fcm_dpo/beta": 0.0030300889629870653, + "fcm_dpo/delta": -0.00830287579447031, + "fcm_dpo/margin": 92.14117431640625, + "fcm_dpo/q_t": 0.43474820256233215, + "grad_norm": 89.22589874267578, + "learning_rate": 6.28723129572247e-08, + "logits/chosen": -0.8947068452835083, + "logits/rejected": -0.8764788508415222, + "logps/chosen": -493.704833984375, + "logps/ref_chosen": -265.0807800292969, + "logps/ref_rejected": -230.58932495117188, + "logps/rejected": -551.3544921875, + "loss": 4.7549, + "margin_dpo/margin_mean": 92.14117431640625, + "margin_dpo/margin_std": 164.5974578857422, + "step": 379 + }, + { + "KL/chosen_KL_mean": -243.8111572265625, + "KL/mean": -299.34429931640625, + "KL/rejected_KL_mean": -354.87744140625, + "KL/std": 164.056640625, + "epoch": 0.7958115183246073, + "fcm_dpo/beta": 0.0029976628720760345, + "fcm_dpo/delta": -0.021797433495521545, + "fcm_dpo/margin": 111.06632995605469, + "fcm_dpo/q_t": 0.4255141019821167, + "grad_norm": 91.8367691040039, + "learning_rate": 6.166331963291519e-08, + "logits/chosen": -0.8527557253837585, + "logits/rejected": -0.8262585997581482, + "logps/chosen": -549.7195434570312, + "logps/ref_chosen": -305.90838623046875, + "logps/ref_rejected": -286.5906677246094, + "logps/rejected": -641.4681396484375, + "loss": 4.6676, + "margin_dpo/margin_mean": 111.06632232666016, + "margin_dpo/margin_std": 190.367919921875, + "step": 380 + }, + { + "KL/chosen_KL_mean": -243.97262573242188, + "KL/mean": -303.4189147949219, + "KL/rejected_KL_mean": -362.86517333984375, + "KL/std": 184.5408477783203, + "epoch": 0.7979057591623037, + "fcm_dpo/beta": 0.002953719347715378, + "fcm_dpo/delta": -0.021564383059740067, + "fcm_dpo/margin": 118.892578125, + "fcm_dpo/q_t": 0.422404944896698, + "grad_norm": 64.48091888427734, + "learning_rate": 6.046442623320145e-08, + "logits/chosen": -0.8266147375106812, + "logits/rejected": -0.7924824357032776, + "logps/chosen": -496.84326171875, + "logps/ref_chosen": -252.87066650390625, + "logps/ref_rejected": -261.1927490234375, + "logps/rejected": -624.0579223632812, + "loss": 4.5793, + "margin_dpo/margin_mean": 118.892578125, + "margin_dpo/margin_std": 191.17279052734375, + "step": 381 + }, + { + "KL/chosen_KL_mean": -236.02357482910156, + "KL/mean": -297.5960998535156, + "KL/rejected_KL_mean": -359.1686706542969, + "KL/std": 151.47557067871094, + "epoch": 0.8, + "fcm_dpo/beta": 0.0028901936020702124, + "fcm_dpo/delta": -0.010631639510393143, + "fcm_dpo/margin": 123.14509582519531, + "fcm_dpo/q_t": 0.4171634614467621, + "grad_norm": 66.18559265136719, + "learning_rate": 5.9275697051098275e-08, + "logits/chosen": -0.8696933388710022, + "logits/rejected": -0.8662832379341125, + "logps/chosen": -525.2350463867188, + "logps/ref_chosen": -289.2114562988281, + "logps/ref_rejected": -278.45751953125, + "logps/rejected": -637.626220703125, + "loss": 4.4974, + "margin_dpo/margin_mean": 123.14509582519531, + "margin_dpo/margin_std": 173.98348999023438, + "step": 382 + }, + { + "KL/chosen_KL_mean": -237.31979370117188, + "KL/mean": -290.00091552734375, + "KL/rejected_KL_mean": -342.68206787109375, + "KL/std": 168.33383178710938, + "epoch": 0.8020942408376963, + "fcm_dpo/beta": 0.0028781089931726456, + "fcm_dpo/delta": -0.025008056312799454, + "fcm_dpo/margin": 105.36231994628906, + "fcm_dpo/q_t": 0.43072545528411865, + "grad_norm": 75.10661315917969, + "learning_rate": 5.809719583454414e-08, + "logits/chosen": -0.850218653678894, + "logits/rejected": -0.8339353203773499, + "logps/chosen": -510.9505920410156, + "logps/ref_chosen": -273.630859375, + "logps/ref_rejected": -261.44024658203125, + "logps/rejected": -604.122314453125, + "loss": 4.7315, + "margin_dpo/margin_mean": 105.36231231689453, + "margin_dpo/margin_std": 192.60629272460938, + "step": 383 + }, + { + "KL/chosen_KL_mean": -241.386474609375, + "KL/mean": -290.0650939941406, + "KL/rejected_KL_mean": -338.7437438964844, + "KL/std": 136.2803192138672, + "epoch": 0.8041884816753927, + "fcm_dpo/beta": 0.0028522873762995005, + "fcm_dpo/delta": 0.014860259369015694, + "fcm_dpo/margin": 97.35729217529297, + "fcm_dpo/q_t": 0.4356921911239624, + "grad_norm": 54.56941223144531, + "learning_rate": 5.6928985782982524e-08, + "logits/chosen": -0.8740838766098022, + "logits/rejected": -0.8694276213645935, + "logps/chosen": -515.9564208984375, + "logps/ref_chosen": -274.5699462890625, + "logps/ref_rejected": -285.8253479003906, + "logps/rejected": -624.569091796875, + "loss": 4.7479, + "margin_dpo/margin_mean": 97.35729217529297, + "margin_dpo/margin_std": 170.97972106933594, + "step": 384 + }, + { + "KL/chosen_KL_mean": -248.646484375, + "KL/mean": -288.6224365234375, + "KL/rejected_KL_mean": -328.598388671875, + "KL/std": 142.26695251464844, + "epoch": 0.806282722513089, + "fcm_dpo/beta": 0.002904432127252221, + "fcm_dpo/delta": 0.019007494673132896, + "fcm_dpo/margin": 79.95191955566406, + "fcm_dpo/q_t": 0.44628220796585083, + "grad_norm": 70.91700744628906, + "learning_rate": 5.57711295439732e-08, + "logits/chosen": -0.8183778524398804, + "logits/rejected": -0.8135970830917358, + "logps/chosen": -532.797119140625, + "logps/ref_chosen": -284.150634765625, + "logps/ref_rejected": -244.87921142578125, + "logps/rejected": -573.4776000976562, + "loss": 4.8742, + "margin_dpo/margin_mean": 79.95191955566406, + "margin_dpo/margin_std": 152.34213256835938, + "step": 385 + }, + { + "KL/chosen_KL_mean": -239.798095703125, + "KL/mean": -304.68206787109375, + "KL/rejected_KL_mean": -369.5660095214844, + "KL/std": 161.51425170898438, + "epoch": 0.8083769633507853, + "fcm_dpo/beta": 0.0028078884351998568, + "fcm_dpo/delta": -0.05693187564611435, + "fcm_dpo/margin": 129.7678985595703, + "fcm_dpo/q_t": 0.4165182411670685, + "grad_norm": 78.76203155517578, + "learning_rate": 5.4623689209832484e-08, + "logits/chosen": -0.7965095639228821, + "logits/rejected": -0.7943077087402344, + "logps/chosen": -559.974365234375, + "logps/ref_chosen": -320.1762390136719, + "logps/ref_rejected": -302.05023193359375, + "logps/rejected": -671.6162109375, + "loss": 4.503, + "margin_dpo/margin_mean": 129.7678985595703, + "margin_dpo/margin_std": 177.5083770751953, + "step": 386 + }, + { + "KL/chosen_KL_mean": -255.77442932128906, + "KL/mean": -303.63458251953125, + "KL/rejected_KL_mean": -351.4947204589844, + "KL/std": 153.2172088623047, + "epoch": 0.8104712041884817, + "fcm_dpo/beta": 0.0027528819628059864, + "fcm_dpo/delta": -0.027687324211001396, + "fcm_dpo/margin": 95.72029113769531, + "fcm_dpo/q_t": 0.43756136298179626, + "grad_norm": 80.19580078125, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": -0.8275306224822998, + "logits/rejected": -0.8312669396400452, + "logps/chosen": -528.0545654296875, + "logps/ref_chosen": -272.2801513671875, + "logps/ref_rejected": -265.1615905761719, + "logps/rejected": -616.6563110351562, + "loss": 4.8415, + "margin_dpo/margin_mean": 95.72029113769531, + "margin_dpo/margin_std": 183.86953735351562, + "step": 387 + }, + { + "KL/chosen_KL_mean": -273.6151123046875, + "KL/mean": -306.04327392578125, + "KL/rejected_KL_mean": -338.471435546875, + "KL/std": 158.35028076171875, + "epoch": 0.812565445026178, + "fcm_dpo/beta": 0.0026982480194419622, + "fcm_dpo/delta": -0.011122354306280613, + "fcm_dpo/margin": 64.85627746582031, + "fcm_dpo/q_t": 0.45950567722320557, + "grad_norm": 84.51703643798828, + "learning_rate": 5.2360301829254745e-08, + "logits/chosen": -0.8161114454269409, + "logits/rejected": -0.8020353317260742, + "logps/chosen": -546.146484375, + "logps/ref_chosen": -272.5313415527344, + "logps/ref_rejected": -239.55735778808594, + "logps/rejected": -578.02880859375, + "loss": 5.1513, + "margin_dpo/margin_mean": 64.85627746582031, + "margin_dpo/margin_std": 188.13427734375, + "step": 388 + }, + { + "KL/chosen_KL_mean": -266.1781005859375, + "KL/mean": -321.2748718261719, + "KL/rejected_KL_mean": -376.37164306640625, + "KL/std": 165.8982391357422, + "epoch": 0.8146596858638744, + "fcm_dpo/beta": 0.0026822565123438835, + "fcm_dpo/delta": -0.018625523895025253, + "fcm_dpo/margin": 110.19355010986328, + "fcm_dpo/q_t": 0.43248510360717773, + "grad_norm": 62.38093948364258, + "learning_rate": 5.1244476161413806e-08, + "logits/chosen": -0.8523331880569458, + "logits/rejected": -0.8490720987319946, + "logps/chosen": -547.267333984375, + "logps/ref_chosen": -281.0892639160156, + "logps/ref_rejected": -246.50045776367188, + "logps/rejected": -622.8720703125, + "loss": 4.7258, + "margin_dpo/margin_mean": 110.19355773925781, + "margin_dpo/margin_std": 189.8052978515625, + "step": 389 + }, + { + "KL/chosen_KL_mean": -259.96453857421875, + "KL/mean": -313.5345458984375, + "KL/rejected_KL_mean": -367.10455322265625, + "KL/std": 155.7751922607422, + "epoch": 0.8167539267015707, + "fcm_dpo/beta": 0.0026583157014101744, + "fcm_dpo/delta": -0.00272680283524096, + "fcm_dpo/margin": 107.14004516601562, + "fcm_dpo/q_t": 0.4331052899360657, + "grad_norm": 59.64759063720703, + "learning_rate": 5.013930914912476e-08, + "logits/chosen": -0.8624210357666016, + "logits/rejected": -0.8728106021881104, + "logps/chosen": -543.9520263671875, + "logps/ref_chosen": -283.98748779296875, + "logps/ref_rejected": -283.465087890625, + "logps/rejected": -650.569580078125, + "loss": 4.7094, + "margin_dpo/margin_mean": 107.14004516601562, + "margin_dpo/margin_std": 176.22068786621094, + "step": 390 + }, + { + "KL/chosen_KL_mean": -267.4212951660156, + "KL/mean": -325.6793518066406, + "KL/rejected_KL_mean": -383.93743896484375, + "KL/std": 176.0823516845703, + "epoch": 0.818848167539267, + "fcm_dpo/beta": 0.002617758931592107, + "fcm_dpo/delta": -0.02285085991024971, + "fcm_dpo/margin": 116.51616668701172, + "fcm_dpo/q_t": 0.42977476119995117, + "grad_norm": 69.12674713134766, + "learning_rate": 4.904486005914027e-08, + "logits/chosen": -0.8110455274581909, + "logits/rejected": -0.7923341393470764, + "logps/chosen": -551.2826538085938, + "logps/ref_chosen": -283.86138916015625, + "logps/ref_rejected": -263.5093688964844, + "logps/rejected": -647.44677734375, + "loss": 4.6727, + "margin_dpo/margin_mean": 116.51615905761719, + "margin_dpo/margin_std": 188.8794708251953, + "step": 391 + }, + { + "KL/chosen_KL_mean": -254.72386169433594, + "KL/mean": -318.38873291015625, + "KL/rejected_KL_mean": -382.05364990234375, + "KL/std": 156.4735870361328, + "epoch": 0.8209424083769633, + "fcm_dpo/beta": 0.0025636577047407627, + "fcm_dpo/delta": -0.012513198889791965, + "fcm_dpo/margin": 127.32978820800781, + "fcm_dpo/q_t": 0.42234691977500916, + "grad_norm": 87.62789916992188, + "learning_rate": 4.796118758344353e-08, + "logits/chosen": -0.8127277493476868, + "logits/rejected": -0.8246631026268005, + "logps/chosen": -564.7939453125, + "logps/ref_chosen": -310.070068359375, + "logps/ref_rejected": -252.89817810058594, + "logps/rejected": -634.9518432617188, + "loss": 4.5707, + "margin_dpo/margin_mean": 127.32978820800781, + "margin_dpo/margin_std": 177.67172241210938, + "step": 392 + }, + { + "KL/chosen_KL_mean": -254.2708740234375, + "KL/mean": -301.04620361328125, + "KL/rejected_KL_mean": -347.821533203125, + "KL/std": 150.13577270507812, + "epoch": 0.8230366492146597, + "fcm_dpo/beta": 0.0025904097128659487, + "fcm_dpo/delta": 0.024187462404370308, + "fcm_dpo/margin": 93.55062103271484, + "fcm_dpo/q_t": 0.4434916079044342, + "grad_norm": 52.19322204589844, + "learning_rate": 4.688834983610082e-08, + "logits/chosen": -0.8437351584434509, + "logits/rejected": -0.8238535523414612, + "logps/chosen": -540.9865112304688, + "logps/ref_chosen": -286.7156677246094, + "logps/ref_rejected": -230.00357055664062, + "logps/rejected": -577.8250732421875, + "loss": 4.8433, + "margin_dpo/margin_mean": 93.55061340332031, + "margin_dpo/margin_std": 174.47572326660156, + "step": 393 + }, + { + "KL/chosen_KL_mean": -257.57318115234375, + "KL/mean": -304.79718017578125, + "KL/rejected_KL_mean": -352.02117919921875, + "KL/std": 179.62074279785156, + "epoch": 0.8251308900523561, + "fcm_dpo/beta": 0.0026478366926312447, + "fcm_dpo/delta": 0.02115212380886078, + "fcm_dpo/margin": 94.44804382324219, + "fcm_dpo/q_t": 0.444901704788208, + "grad_norm": 57.13371658325195, + "learning_rate": 4.582640435014459e-08, + "logits/chosen": -0.8858999609947205, + "logits/rejected": -0.875397801399231, + "logps/chosen": -583.566650390625, + "logps/ref_chosen": -325.9934387207031, + "logps/ref_rejected": -317.42706298828125, + "logps/rejected": -669.4482421875, + "loss": 4.9197, + "margin_dpo/margin_mean": 94.44804382324219, + "margin_dpo/margin_std": 205.414306640625, + "step": 394 + }, + { + "KL/chosen_KL_mean": -244.72979736328125, + "KL/mean": -310.8471984863281, + "KL/rejected_KL_mean": -376.9646301269531, + "KL/std": 167.30059814453125, + "epoch": 0.8272251308900523, + "fcm_dpo/beta": 0.002669533481821418, + "fcm_dpo/delta": -0.015710730105638504, + "fcm_dpo/margin": 132.23486328125, + "fcm_dpo/q_t": 0.4210633933544159, + "grad_norm": 63.38975524902344, + "learning_rate": 4.477540807448832e-08, + "logits/chosen": -0.8237651586532593, + "logits/rejected": -0.8312820792198181, + "logps/chosen": -513.630615234375, + "logps/ref_chosen": -268.90081787109375, + "logps/ref_rejected": -272.85809326171875, + "logps/rejected": -649.82275390625, + "loss": 4.5299, + "margin_dpo/margin_mean": 132.23486328125, + "margin_dpo/margin_std": 197.66409301757812, + "step": 395 + }, + { + "KL/chosen_KL_mean": -267.392333984375, + "KL/mean": -313.00030517578125, + "KL/rejected_KL_mean": -358.6082763671875, + "KL/std": 159.95083618164062, + "epoch": 0.8293193717277487, + "fcm_dpo/beta": 0.002658443059772253, + "fcm_dpo/delta": 0.013091212138533592, + "fcm_dpo/margin": 91.2159194946289, + "fcm_dpo/q_t": 0.443788081407547, + "grad_norm": 103.75393676757812, + "learning_rate": 4.373541737087263e-08, + "logits/chosen": -0.833990752696991, + "logits/rejected": -0.8136544823646545, + "logps/chosen": -558.5906372070312, + "logps/ref_chosen": -291.19830322265625, + "logps/ref_rejected": -253.2803955078125, + "logps/rejected": -611.888671875, + "loss": 4.8872, + "margin_dpo/margin_mean": 91.2159194946289, + "margin_dpo/margin_std": 185.2631072998047, + "step": 396 + }, + { + "KL/chosen_KL_mean": -252.65269470214844, + "KL/mean": -297.8766174316406, + "KL/rejected_KL_mean": -343.1005859375, + "KL/std": 157.75360107421875, + "epoch": 0.831413612565445, + "fcm_dpo/beta": 0.002626030705869198, + "fcm_dpo/delta": -0.026181943714618683, + "fcm_dpo/margin": 90.44788360595703, + "fcm_dpo/q_t": 0.4456873834133148, + "grad_norm": 70.29791259765625, + "learning_rate": 4.270648801084295e-08, + "logits/chosen": -0.8465428352355957, + "logits/rejected": -0.8269012570381165, + "logps/chosen": -562.47509765625, + "logps/ref_chosen": -309.8224182128906, + "logps/ref_rejected": -291.9057922363281, + "logps/rejected": -635.00634765625, + "loss": 4.9391, + "margin_dpo/margin_mean": 90.44788360595703, + "margin_dpo/margin_std": 190.6138153076172, + "step": 397 + }, + { + "KL/chosen_KL_mean": -283.8010559082031, + "KL/mean": -324.195068359375, + "KL/rejected_KL_mean": -364.5890197753906, + "KL/std": 173.2881317138672, + "epoch": 0.8335078534031414, + "fcm_dpo/beta": 0.0025994465686380863, + "fcm_dpo/delta": -0.006987897679209709, + "fcm_dpo/margin": 80.7879638671875, + "fcm_dpo/q_t": 0.44956809282302856, + "grad_norm": 79.59461212158203, + "learning_rate": 4.168867517275806e-08, + "logits/chosen": -0.7517906427383423, + "logits/rejected": -0.7874580025672913, + "logps/chosen": -581.6146240234375, + "logps/ref_chosen": -297.8135070800781, + "logps/ref_rejected": -270.5025634765625, + "logps/rejected": -635.0916137695312, + "loss": 5.1179, + "margin_dpo/margin_mean": 80.78797149658203, + "margin_dpo/margin_std": 224.45187377929688, + "step": 398 + }, + { + "KL/chosen_KL_mean": -264.94573974609375, + "KL/mean": -315.5485534667969, + "KL/rejected_KL_mean": -366.15130615234375, + "KL/std": 161.73268127441406, + "epoch": 0.8356020942408376, + "fcm_dpo/beta": 0.0026253818068653345, + "fcm_dpo/delta": 0.022293759509921074, + "fcm_dpo/margin": 101.20550537109375, + "fcm_dpo/q_t": 0.4391327202320099, + "grad_norm": 73.089111328125, + "learning_rate": 4.0682033438831584e-08, + "logits/chosen": -0.8615554571151733, + "logits/rejected": -0.8228447437286377, + "logps/chosen": -557.7925415039062, + "logps/ref_chosen": -292.8467712402344, + "logps/ref_rejected": -268.3638916015625, + "logps/rejected": -634.51513671875, + "loss": 4.8136, + "margin_dpo/margin_mean": 101.20550537109375, + "margin_dpo/margin_std": 190.82528686523438, + "step": 399 + }, + { + "KL/chosen_KL_mean": -255.96389770507812, + "KL/mean": -306.29583740234375, + "KL/rejected_KL_mean": -356.6277770996094, + "KL/std": 157.36746215820312, + "epoch": 0.837696335078534, + "fcm_dpo/beta": 0.002646203152835369, + "fcm_dpo/delta": 0.005138233304023743, + "fcm_dpo/margin": 100.66392517089844, + "fcm_dpo/q_t": 0.43769270181655884, + "grad_norm": 51.399471282958984, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": -0.8845930695533752, + "logits/rejected": -0.8838775753974915, + "logps/chosen": -519.6402587890625, + "logps/ref_chosen": -263.6763916015625, + "logps/ref_rejected": -258.67266845703125, + "logps/rejected": -615.3004760742188, + "loss": 4.8145, + "margin_dpo/margin_mean": 100.66392517089844, + "margin_dpo/margin_std": 187.87037658691406, + "step": 400 + }, + { + "epoch": 0.837696335078534, + "eval_KL/chosen_KL_mean": -259.0677185058594, + "eval_KL/mean": -311.7682800292969, + "eval_KL/rejected_KL_mean": -364.4688720703125, + "eval_KL/std": 162.36822509765625, + "eval_fcm_dpo/beta": 0.0026530208997428417, + "eval_logits/chosen": -0.8539494872093201, + "eval_logits/rejected": -0.8364132642745972, + "eval_logps/chosen": -546.8944702148438, + "eval_logps/ref_chosen": -287.8268127441406, + "eval_logps/ref_rejected": -266.9300231933594, + "eval_logps/rejected": -631.3988647460938, + "eval_loss": 0.5982628464698792, + "eval_margin_dpo/margin_mean": 105.40113830566406, + "eval_margin_dpo/margin_std": 182.74659729003906, + "eval_runtime": 78.7931, + "eval_samples_per_second": 25.383, + "eval_steps_per_second": 3.173, + "step": 400 + }, + { + "KL/chosen_KL_mean": -256.0751953125, + "KL/mean": -309.08087158203125, + "KL/rejected_KL_mean": -362.0865478515625, + "KL/std": 163.9885711669922, + "epoch": 0.8397905759162304, + "fcm_dpo/beta": 0.002689874265342951, + "fcm_dpo/delta": 0.024891149252653122, + "fcm_dpo/margin": 106.01134490966797, + "fcm_dpo/q_t": 0.4338783919811249, + "grad_norm": 69.87895202636719, + "learning_rate": 3.8702478614051345e-08, + "logits/chosen": -0.83061283826828, + "logits/rejected": -0.8284586668014526, + "logps/chosen": -574.3605346679688, + "logps/ref_chosen": -318.2853088378906, + "logps/ref_rejected": -293.75225830078125, + "logps/rejected": -655.8388061523438, + "loss": 4.7405, + "margin_dpo/margin_mean": 106.01134490966797, + "margin_dpo/margin_std": 185.73927307128906, + "step": 401 + }, + { + "KL/chosen_KL_mean": -247.47999572753906, + "KL/mean": -295.99700927734375, + "KL/rejected_KL_mean": -344.5140075683594, + "KL/std": 166.7805938720703, + "epoch": 0.8418848167539267, + "fcm_dpo/beta": 0.0027063796296715736, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 97.03401184082031, + "fcm_dpo/q_t": 0.4415395259857178, + "grad_norm": 67.62411499023438, + "learning_rate": 3.772967168071517e-08, + "logits/chosen": -0.8851345181465149, + "logits/rejected": -0.8621891736984253, + "logps/chosen": -556.9078369140625, + "logps/ref_chosen": -309.4278564453125, + "logps/ref_rejected": -282.0279846191406, + "logps/rejected": -626.5419921875, + "loss": 4.8518, + "margin_dpo/margin_mean": 97.03401184082031, + "margin_dpo/margin_std": 202.4774169921875, + "step": 402 + }, + { + "KL/chosen_KL_mean": -236.76856994628906, + "KL/mean": -308.6864318847656, + "KL/rejected_KL_mean": -380.604248046875, + "KL/std": 171.2498779296875, + "epoch": 0.8439790575916231, + "fcm_dpo/beta": 0.002709330292418599, + "fcm_dpo/delta": -0.025865260511636734, + "fcm_dpo/margin": 143.83566284179688, + "fcm_dpo/q_t": 0.41079238057136536, + "grad_norm": 60.24233627319336, + "learning_rate": 3.676824816087978e-08, + "logits/chosen": -0.8610972166061401, + "logits/rejected": -0.835875928401947, + "logps/chosen": -545.7970581054688, + "logps/ref_chosen": -309.0284729003906, + "logps/ref_rejected": -272.9622497558594, + "logps/rejected": -653.5665283203125, + "loss": 4.3735, + "margin_dpo/margin_mean": 143.83567810058594, + "margin_dpo/margin_std": 179.44992065429688, + "step": 403 + }, + { + "KL/chosen_KL_mean": -261.0257873535156, + "KL/mean": -315.1200256347656, + "KL/rejected_KL_mean": -369.2142028808594, + "KL/std": 173.9381866455078, + "epoch": 0.8460732984293193, + "fcm_dpo/beta": 0.002677470212802291, + "fcm_dpo/delta": 0.0036769555881619453, + "fcm_dpo/margin": 108.18839263916016, + "fcm_dpo/q_t": 0.4339839220046997, + "grad_norm": 78.65907287597656, + "learning_rate": 3.581825961277074e-08, + "logits/chosen": -0.8783624172210693, + "logits/rejected": -0.8541516661643982, + "logps/chosen": -558.3095703125, + "logps/ref_chosen": -297.2837219238281, + "logps/ref_rejected": -256.99041748046875, + "logps/rejected": -626.20458984375, + "loss": 4.7808, + "margin_dpo/margin_mean": 108.18839263916016, + "margin_dpo/margin_std": 206.70828247070312, + "step": 404 + }, + { + "KL/chosen_KL_mean": -249.36727905273438, + "KL/mean": -309.5423889160156, + "KL/rejected_KL_mean": -369.7174987792969, + "KL/std": 169.79800415039062, + "epoch": 0.8481675392670157, + "fcm_dpo/beta": 0.002700282959267497, + "fcm_dpo/delta": 0.026766503229737282, + "fcm_dpo/margin": 120.3502197265625, + "fcm_dpo/q_t": 0.42523056268692017, + "grad_norm": 61.90435028076172, + "learning_rate": 3.487975698139084e-08, + "logits/chosen": -0.8048846125602722, + "logits/rejected": -0.8137521147727966, + "logps/chosen": -507.3326110839844, + "logps/ref_chosen": -257.96533203125, + "logps/ref_rejected": -255.811279296875, + "logps/rejected": -625.52880859375, + "loss": 4.6128, + "margin_dpo/margin_mean": 120.3502197265625, + "margin_dpo/margin_std": 191.45315551757812, + "step": 405 + }, + { + "KL/chosen_KL_mean": -269.34295654296875, + "KL/mean": -309.04095458984375, + "KL/rejected_KL_mean": -348.739013671875, + "KL/std": 154.44068908691406, + "epoch": 0.8502617801047121, + "fcm_dpo/beta": 0.002717172261327505, + "fcm_dpo/delta": 0.003170239506289363, + "fcm_dpo/margin": 79.39606475830078, + "fcm_dpo/q_t": 0.4483673572540283, + "grad_norm": 113.7933349609375, + "learning_rate": 3.3952790595787986e-08, + "logits/chosen": -0.8405861258506775, + "logits/rejected": -0.8171727061271667, + "logps/chosen": -554.5239868164062, + "logps/ref_chosen": -285.1810607910156, + "logps/ref_rejected": -264.41351318359375, + "logps/rejected": -613.1525268554688, + "loss": 4.9484, + "margin_dpo/margin_mean": 79.39606475830078, + "margin_dpo/margin_std": 170.13916015625, + "step": 406 + }, + { + "KL/chosen_KL_mean": -265.46820068359375, + "KL/mean": -321.43084716796875, + "KL/rejected_KL_mean": -377.3935546875, + "KL/std": 168.72860717773438, + "epoch": 0.8523560209424084, + "fcm_dpo/beta": 0.0027630478143692017, + "fcm_dpo/delta": 0.02725648693740368, + "fcm_dpo/margin": 111.92539978027344, + "fcm_dpo/q_t": 0.4298943281173706, + "grad_norm": 55.85504913330078, + "learning_rate": 3.303741016635614e-08, + "logits/chosen": -0.8258509635925293, + "logits/rejected": -0.8421826362609863, + "logps/chosen": -530.706298828125, + "logps/ref_chosen": -265.23809814453125, + "logps/ref_rejected": -219.0631561279297, + "logps/rejected": -596.4567260742188, + "loss": 4.7059, + "margin_dpo/margin_mean": 111.9253921508789, + "margin_dpo/margin_std": 197.7037353515625, + "step": 407 + }, + { + "KL/chosen_KL_mean": -247.27427673339844, + "KL/mean": -302.8316345214844, + "KL/rejected_KL_mean": -358.3890686035156, + "KL/std": 157.02703857421875, + "epoch": 0.8544502617801047, + "fcm_dpo/beta": 0.0027716769836843014, + "fcm_dpo/delta": -0.04041987285017967, + "fcm_dpo/margin": 111.11476135253906, + "fcm_dpo/q_t": 0.42840081453323364, + "grad_norm": 79.1960678100586, + "learning_rate": 3.2133664782169944e-08, + "logits/chosen": -0.8630377054214478, + "logits/rejected": -0.8538559079170227, + "logps/chosen": -544.2468872070312, + "logps/ref_chosen": -296.9726257324219, + "logps/ref_rejected": -295.4786376953125, + "logps/rejected": -653.8676147460938, + "loss": 4.6593, + "margin_dpo/margin_mean": 111.11476135253906, + "margin_dpo/margin_std": 172.04241943359375, + "step": 408 + }, + { + "KL/chosen_KL_mean": -260.24188232421875, + "KL/mean": -322.407958984375, + "KL/rejected_KL_mean": -384.5740051269531, + "KL/std": 207.52333068847656, + "epoch": 0.856544502617801, + "fcm_dpo/beta": 0.002639027778059244, + "fcm_dpo/delta": -0.04701896011829376, + "fcm_dpo/margin": 124.33211517333984, + "fcm_dpo/q_t": 0.4329832196235657, + "grad_norm": 53.74481201171875, + "learning_rate": 3.12416029083514e-08, + "logits/chosen": -0.839493989944458, + "logits/rejected": -0.8232097029685974, + "logps/chosen": -547.6212768554688, + "logps/ref_chosen": -287.37933349609375, + "logps/ref_rejected": -275.80291748046875, + "logps/rejected": -660.3768310546875, + "loss": 4.759, + "margin_dpo/margin_mean": 124.33211517333984, + "margin_dpo/margin_std": 254.77313232421875, + "step": 409 + }, + { + "KL/chosen_KL_mean": -251.41685485839844, + "KL/mean": -302.3774108886719, + "KL/rejected_KL_mean": -353.33795166015625, + "KL/std": 163.56126403808594, + "epoch": 0.8586387434554974, + "fcm_dpo/beta": 0.0026220958679914474, + "fcm_dpo/delta": 0.018517207354307175, + "fcm_dpo/margin": 101.92111206054688, + "fcm_dpo/q_t": 0.43905141949653625, + "grad_norm": 59.67084884643555, + "learning_rate": 3.036127238347164e-08, + "logits/chosen": -0.8541938662528992, + "logits/rejected": -0.8597331047058105, + "logps/chosen": -533.197021484375, + "logps/ref_chosen": -281.7801818847656, + "logps/ref_rejected": -266.7550354003906, + "logps/rejected": -620.093017578125, + "loss": 4.8163, + "margin_dpo/margin_mean": 101.92111206054688, + "margin_dpo/margin_std": 195.57972717285156, + "step": 410 + }, + { + "KL/chosen_KL_mean": -244.28140258789062, + "KL/mean": -301.5091857910156, + "KL/rejected_KL_mean": -358.7369079589844, + "KL/std": 168.92539978027344, + "epoch": 0.8607329842931937, + "fcm_dpo/beta": 0.0026748264208436012, + "fcm_dpo/delta": 0.0014014113694429398, + "fcm_dpo/margin": 114.45552825927734, + "fcm_dpo/q_t": 0.4303073585033417, + "grad_norm": 85.47948455810547, + "learning_rate": 2.9492720416985e-08, + "logits/chosen": -0.8583596348762512, + "logits/rejected": -0.8106494545936584, + "logps/chosen": -525.86865234375, + "logps/ref_chosen": -281.5872497558594, + "logps/ref_rejected": -254.78916931152344, + "logps/rejected": -613.526123046875, + "loss": 4.6366, + "margin_dpo/margin_mean": 114.45552825927734, + "margin_dpo/margin_std": 170.4075927734375, + "step": 411 + }, + { + "KL/chosen_KL_mean": -273.9906921386719, + "KL/mean": -319.8974304199219, + "KL/rejected_KL_mean": -365.8041687011719, + "KL/std": 163.98312377929688, + "epoch": 0.86282722513089, + "fcm_dpo/beta": 0.002576531609520316, + "fcm_dpo/delta": -0.026781653985381126, + "fcm_dpo/margin": 91.8134994506836, + "fcm_dpo/q_t": 0.44552770256996155, + "grad_norm": 70.79202270507812, + "learning_rate": 2.863599358669755e-08, + "logits/chosen": -0.8241173624992371, + "logits/rejected": -0.8364276885986328, + "logps/chosen": -550.5248413085938, + "logps/ref_chosen": -276.5341796875, + "logps/ref_rejected": -273.8751220703125, + "logps/rejected": -639.6793212890625, + "loss": 4.9052, + "margin_dpo/margin_mean": 91.8134994506836, + "margin_dpo/margin_std": 183.9867401123047, + "step": 412 + }, + { + "KL/chosen_KL_mean": -252.1434783935547, + "KL/mean": -297.56536865234375, + "KL/rejected_KL_mean": -342.98724365234375, + "KL/std": 162.1449737548828, + "epoch": 0.8649214659685864, + "fcm_dpo/beta": 0.0025809926446527243, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 90.84378051757812, + "fcm_dpo/q_t": 0.44537127017974854, + "grad_norm": 60.38282775878906, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": -0.840187132358551, + "logits/rejected": -0.8357990384101868, + "logps/chosen": -523.4180297851562, + "logps/ref_chosen": -271.2745666503906, + "logps/ref_rejected": -270.16912841796875, + "logps/rejected": -613.1563720703125, + "loss": 4.8861, + "margin_dpo/margin_mean": 90.84378051757812, + "margin_dpo/margin_std": 185.5474395751953, + "step": 413 + }, + { + "KL/chosen_KL_mean": -254.0613250732422, + "KL/mean": -316.70928955078125, + "KL/rejected_KL_mean": -379.35723876953125, + "KL/std": 163.72683715820312, + "epoch": 0.8670157068062827, + "fcm_dpo/beta": 0.0025707168970257044, + "fcm_dpo/delta": -0.01698092371225357, + "fcm_dpo/margin": 125.29594421386719, + "fcm_dpo/q_t": 0.4260947108268738, + "grad_norm": 62.23833084106445, + "learning_rate": 2.6958198472749717e-08, + "logits/chosen": -0.8793128728866577, + "logits/rejected": -0.872239351272583, + "logps/chosen": -551.1763305664062, + "logps/ref_chosen": -297.11505126953125, + "logps/ref_rejected": -271.7034606933594, + "logps/rejected": -651.060791015625, + "loss": 4.6003, + "margin_dpo/margin_mean": 125.29594421386719, + "margin_dpo/margin_std": 187.06222534179688, + "step": 414 + }, + { + "KL/chosen_KL_mean": -251.2029266357422, + "KL/mean": -303.4581604003906, + "KL/rejected_KL_mean": -355.71331787109375, + "KL/std": 149.83013916015625, + "epoch": 0.8691099476439791, + "fcm_dpo/beta": 0.0025737180840224028, + "fcm_dpo/delta": 0.01915055885910988, + "fcm_dpo/margin": 104.51042938232422, + "fcm_dpo/q_t": 0.4367385804653168, + "grad_norm": 63.56855773925781, + "learning_rate": 2.613722016414943e-08, + "logits/chosen": -0.871803343296051, + "logits/rejected": -0.8581423163414001, + "logps/chosen": -548.8956298828125, + "logps/ref_chosen": -297.6926574707031, + "logps/ref_rejected": -279.0503234863281, + "logps/rejected": -634.7637329101562, + "loss": 4.7351, + "margin_dpo/margin_mean": 104.51043701171875, + "margin_dpo/margin_std": 170.8563995361328, + "step": 415 + }, + { + "KL/chosen_KL_mean": -236.1012725830078, + "KL/mean": -297.1469421386719, + "KL/rejected_KL_mean": -358.192626953125, + "KL/std": 151.40481567382812, + "epoch": 0.8712041884816754, + "fcm_dpo/beta": 0.002560949884355068, + "fcm_dpo/delta": -5.1200266170781106e-05, + "fcm_dpo/margin": 122.09134674072266, + "fcm_dpo/q_t": 0.42640596628189087, + "grad_norm": 39.975067138671875, + "learning_rate": 2.5328246937043525e-08, + "logits/chosen": -0.8686580061912537, + "logits/rejected": -0.8653780817985535, + "logps/chosen": -547.9268188476562, + "logps/ref_chosen": -311.8255615234375, + "logps/ref_rejected": -268.6170654296875, + "logps/rejected": -626.8096923828125, + "loss": 4.6017, + "margin_dpo/margin_mean": 122.09133911132812, + "margin_dpo/margin_std": 176.49038696289062, + "step": 416 + }, + { + "KL/chosen_KL_mean": -256.92279052734375, + "KL/mean": -309.4592590332031, + "KL/rejected_KL_mean": -361.9957275390625, + "KL/std": 169.861083984375, + "epoch": 0.8732984293193717, + "fcm_dpo/beta": 0.0025655170902609825, + "fcm_dpo/delta": -0.02516508474946022, + "fcm_dpo/margin": 105.07295227050781, + "fcm_dpo/q_t": 0.43865251541137695, + "grad_norm": 58.286231994628906, + "learning_rate": 2.4531322174210973e-08, + "logits/chosen": -0.824512779712677, + "logits/rejected": -0.8112680315971375, + "logps/chosen": -567.3595581054688, + "logps/ref_chosen": -310.43682861328125, + "logps/ref_rejected": -277.15283203125, + "logps/rejected": -639.1485595703125, + "loss": 4.8026, + "margin_dpo/margin_mean": 105.07294464111328, + "margin_dpo/margin_std": 191.708740234375, + "step": 417 + }, + { + "KL/chosen_KL_mean": -271.6235046386719, + "KL/mean": -324.97332763671875, + "KL/rejected_KL_mean": -378.3231506347656, + "KL/std": 181.0856170654297, + "epoch": 0.875392670157068, + "fcm_dpo/beta": 0.0025076887104660273, + "fcm_dpo/delta": -0.01675930619239807, + "fcm_dpo/margin": 106.69969177246094, + "fcm_dpo/q_t": 0.43919453024864197, + "grad_norm": 80.68965911865234, + "learning_rate": 2.3746488612308295e-08, + "logits/chosen": -0.8077826499938965, + "logits/rejected": -0.7858579158782959, + "logps/chosen": -550.1194458007812, + "logps/ref_chosen": -278.49591064453125, + "logps/ref_rejected": -276.56671142578125, + "logps/rejected": -654.889892578125, + "loss": 4.8247, + "margin_dpo/margin_mean": 106.69967651367188, + "margin_dpo/margin_std": 208.5244598388672, + "step": 418 + }, + { + "KL/chosen_KL_mean": -267.290771484375, + "KL/mean": -320.2489013671875, + "KL/rejected_KL_mean": -373.2070007324219, + "KL/std": 170.7522430419922, + "epoch": 0.8774869109947644, + "fcm_dpo/beta": 0.0025136300828307867, + "fcm_dpo/delta": 0.009540699422359467, + "fcm_dpo/margin": 105.91624450683594, + "fcm_dpo/q_t": 0.4366569519042969, + "grad_norm": 75.40731048583984, + "learning_rate": 2.297378833957761e-08, + "logits/chosen": -0.8613269329071045, + "logits/rejected": -0.8316982388496399, + "logps/chosen": -566.1910400390625, + "logps/ref_chosen": -298.9002380371094, + "logps/ref_rejected": -246.1540985107422, + "logps/rejected": -619.3611450195312, + "loss": 4.8257, + "margin_dpo/margin_mean": 105.91624450683594, + "margin_dpo/margin_std": 208.2015838623047, + "step": 419 + }, + { + "KL/chosen_KL_mean": -260.46661376953125, + "KL/mean": -325.86968994140625, + "KL/rejected_KL_mean": -391.2727966308594, + "KL/std": 182.0087432861328, + "epoch": 0.8795811518324608, + "fcm_dpo/beta": 0.0025134901516139507, + "fcm_dpo/delta": -0.011981809511780739, + "fcm_dpo/margin": 130.80618286132812, + "fcm_dpo/q_t": 0.4250591993331909, + "grad_norm": 66.69501495361328, + "learning_rate": 2.2213262793589482e-08, + "logits/chosen": -0.8003321290016174, + "logits/rejected": -0.771695613861084, + "logps/chosen": -525.0275268554688, + "logps/ref_chosen": -264.5608825683594, + "logps/ref_rejected": -245.67031860351562, + "logps/rejected": -636.9431762695312, + "loss": 4.6755, + "margin_dpo/margin_mean": 130.80618286132812, + "margin_dpo/margin_std": 231.65260314941406, + "step": 420 + }, + { + "KL/chosen_KL_mean": -239.69073486328125, + "KL/mean": -296.38812255859375, + "KL/rejected_KL_mean": -353.08551025390625, + "KL/std": 154.60948181152344, + "epoch": 0.881675392670157, + "fcm_dpo/beta": 0.002471539657562971, + "fcm_dpo/delta": -0.02126682549715042, + "fcm_dpo/margin": 113.39478302001953, + "fcm_dpo/q_t": 0.4333392381668091, + "grad_norm": 67.4413070678711, + "learning_rate": 2.1464952759020856e-08, + "logits/chosen": -0.8876159191131592, + "logits/rejected": -0.8631241917610168, + "logps/chosen": -537.395751953125, + "logps/ref_chosen": -297.70501708984375, + "logps/ref_rejected": -243.74771118164062, + "logps/rejected": -596.8331909179688, + "loss": 4.7061, + "margin_dpo/margin_mean": 113.394775390625, + "margin_dpo/margin_std": 176.9335479736328, + "step": 421 + }, + { + "KL/chosen_KL_mean": -253.8042449951172, + "KL/mean": -306.0145263671875, + "KL/rejected_KL_mean": -358.22479248046875, + "KL/std": 167.76966857910156, + "epoch": 0.8837696335078534, + "fcm_dpo/beta": 0.002462256234139204, + "fcm_dpo/delta": 0.015522689558565617, + "fcm_dpo/margin": 104.4205322265625, + "fcm_dpo/q_t": 0.44079747796058655, + "grad_norm": 51.60731887817383, + "learning_rate": 2.07288983654679e-08, + "logits/chosen": -0.7430369257926941, + "logits/rejected": -0.7820625901222229, + "logps/chosen": -542.1630249023438, + "logps/ref_chosen": -288.3587646484375, + "logps/ref_rejected": -256.4377746582031, + "logps/rejected": -614.66259765625, + "loss": 4.805, + "margin_dpo/margin_mean": 104.42054748535156, + "margin_dpo/margin_std": 192.31918334960938, + "step": 422 + }, + { + "KL/chosen_KL_mean": -258.586181640625, + "KL/mean": -319.3099670410156, + "KL/rejected_KL_mean": -380.03375244140625, + "KL/std": 160.17820739746094, + "epoch": 0.8858638743455497, + "fcm_dpo/beta": 0.0024836647789925337, + "fcm_dpo/delta": 0.003166984301060438, + "fcm_dpo/margin": 121.4476318359375, + "fcm_dpo/q_t": 0.42924964427948, + "grad_norm": 62.990482330322266, + "learning_rate": 2.0005139085293942e-08, + "logits/chosen": -0.8718672394752502, + "logits/rejected": -0.849821925163269, + "logps/chosen": -554.5932006835938, + "logps/ref_chosen": -296.00701904296875, + "logps/ref_rejected": -261.3480529785156, + "logps/rejected": -641.3818359375, + "loss": 4.6526, + "margin_dpo/margin_mean": 121.44761657714844, + "margin_dpo/margin_std": 191.81884765625, + "step": 423 + }, + { + "KL/chosen_KL_mean": -238.3405303955078, + "KL/mean": -292.3135986328125, + "KL/rejected_KL_mean": -346.28668212890625, + "KL/std": 150.8682098388672, + "epoch": 0.8879581151832461, + "fcm_dpo/beta": 0.0024709829594939947, + "fcm_dpo/delta": -0.0036196252331137657, + "fcm_dpo/margin": 107.9461669921875, + "fcm_dpo/q_t": 0.4369615614414215, + "grad_norm": 80.2655258178711, + "learning_rate": 1.9293713731512673e-08, + "logits/chosen": -0.8483703136444092, + "logits/rejected": -0.8329979777336121, + "logps/chosen": -547.7623901367188, + "logps/ref_chosen": -309.421875, + "logps/ref_rejected": -249.14886474609375, + "logps/rejected": -595.4356079101562, + "loss": 4.734, + "margin_dpo/margin_mean": 107.9461669921875, + "margin_dpo/margin_std": 167.56033325195312, + "step": 424 + }, + { + "KL/chosen_KL_mean": -247.47341918945312, + "KL/mean": -297.63507080078125, + "KL/rejected_KL_mean": -347.79669189453125, + "KL/std": 160.0552520751953, + "epoch": 0.8900523560209425, + "fcm_dpo/beta": 0.0024773648474365473, + "fcm_dpo/delta": -0.0067837778478860855, + "fcm_dpo/margin": 100.32325744628906, + "fcm_dpo/q_t": 0.44239264726638794, + "grad_norm": 47.60661697387695, + "learning_rate": 1.8594660455706763e-08, + "logits/chosen": -0.8335504531860352, + "logits/rejected": -0.8358312249183655, + "logps/chosen": -527.9825439453125, + "logps/ref_chosen": -280.50909423828125, + "logps/ref_rejected": -276.8252258300781, + "logps/rejected": -624.6218872070312, + "loss": 4.8279, + "margin_dpo/margin_mean": 100.32325744628906, + "margin_dpo/margin_std": 179.0926971435547, + "step": 425 + }, + { + "KL/chosen_KL_mean": -253.93557739257812, + "KL/mean": -309.0635986328125, + "KL/rejected_KL_mean": -364.19158935546875, + "KL/std": 163.70797729492188, + "epoch": 0.8921465968586387, + "fcm_dpo/beta": 0.002501498442143202, + "fcm_dpo/delta": 0.027417544275522232, + "fcm_dpo/margin": 110.25602722167969, + "fcm_dpo/q_t": 0.4352967441082001, + "grad_norm": 61.759361267089844, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": -0.8617511987686157, + "logits/rejected": -0.8399728536605835, + "logps/chosen": -546.7207641601562, + "logps/ref_chosen": -292.78521728515625, + "logps/ref_rejected": -255.62698364257812, + "logps/rejected": -619.8185424804688, + "loss": 4.733, + "margin_dpo/margin_mean": 110.25602722167969, + "margin_dpo/margin_std": 181.19354248046875, + "step": 426 + }, + { + "KL/chosen_KL_mean": -242.61668395996094, + "KL/mean": -321.30316162109375, + "KL/rejected_KL_mean": -399.9896240234375, + "KL/std": 182.28509521484375, + "epoch": 0.8942408376963351, + "fcm_dpo/beta": 0.002506428863853216, + "fcm_dpo/delta": -0.04454171657562256, + "fcm_dpo/margin": 157.37294006347656, + "fcm_dpo/q_t": 0.40898868441581726, + "grad_norm": 74.9897232055664, + "learning_rate": 1.7233819424956247e-08, + "logits/chosen": -0.8332107663154602, + "logits/rejected": -0.7955859303474426, + "logps/chosen": -531.3854370117188, + "logps/ref_chosen": -288.7687072753906, + "logps/ref_rejected": -268.4986572265625, + "logps/rejected": -668.48828125, + "loss": 4.4275, + "margin_dpo/margin_mean": 157.37295532226562, + "margin_dpo/margin_std": 218.2144775390625, + "step": 427 + }, + { + "KL/chosen_KL_mean": -274.7869873046875, + "KL/mean": -343.9637451171875, + "KL/rejected_KL_mean": -413.14044189453125, + "KL/std": 183.8684539794922, + "epoch": 0.8963350785340314, + "fcm_dpo/beta": 0.0024281598161906004, + "fcm_dpo/delta": -0.008501682430505753, + "fcm_dpo/margin": 138.35345458984375, + "fcm_dpo/q_t": 0.42190542817115784, + "grad_norm": 59.844512939453125, + "learning_rate": 1.6572104647786245e-08, + "logits/chosen": -0.791337788105011, + "logits/rejected": -0.8210979700088501, + "logps/chosen": -570.3079833984375, + "logps/ref_chosen": -295.5209655761719, + "logps/ref_rejected": -275.71026611328125, + "logps/rejected": -688.8507080078125, + "loss": 4.5482, + "margin_dpo/margin_mean": 138.35345458984375, + "margin_dpo/margin_std": 195.07057189941406, + "step": 428 + }, + { + "KL/chosen_KL_mean": -264.47540283203125, + "KL/mean": -318.24371337890625, + "KL/rejected_KL_mean": -372.01202392578125, + "KL/std": 177.7375946044922, + "epoch": 0.8984293193717278, + "fcm_dpo/beta": 0.0024204240180552006, + "fcm_dpo/delta": 0.012468225322663784, + "fcm_dpo/margin": 107.53662109375, + "fcm_dpo/q_t": 0.4388344883918762, + "grad_norm": 59.98493194580078, + "learning_rate": 1.5922907900227017e-08, + "logits/chosen": -0.7992820739746094, + "logits/rejected": -0.8066202402114868, + "logps/chosen": -538.8677368164062, + "logps/ref_chosen": -274.392333984375, + "logps/ref_rejected": -258.574462890625, + "logps/rejected": -630.5865478515625, + "loss": 4.862, + "margin_dpo/margin_mean": 107.53662109375, + "margin_dpo/margin_std": 216.66375732421875, + "step": 429 + }, + { + "KL/chosen_KL_mean": -253.07960510253906, + "KL/mean": -308.39447021484375, + "KL/rejected_KL_mean": -363.7093505859375, + "KL/std": 166.74659729003906, + "epoch": 0.900523560209424, + "fcm_dpo/beta": 0.0024694507010281086, + "fcm_dpo/delta": 0.014632913284003735, + "fcm_dpo/margin": 110.62974548339844, + "fcm_dpo/q_t": 0.43679171800613403, + "grad_norm": 72.91577911376953, + "learning_rate": 1.5286263996730026e-08, + "logits/chosen": -0.8794166445732117, + "logits/rejected": -0.847719132900238, + "logps/chosen": -541.8187255859375, + "logps/ref_chosen": -288.7391357421875, + "logps/ref_rejected": -268.6106262207031, + "logps/rejected": -632.3199462890625, + "loss": 4.756, + "margin_dpo/margin_mean": 110.62974548339844, + "margin_dpo/margin_std": 189.39857482910156, + "step": 430 + }, + { + "KL/chosen_KL_mean": -250.58224487304688, + "KL/mean": -295.65850830078125, + "KL/rejected_KL_mean": -340.7347106933594, + "KL/std": 156.20620727539062, + "epoch": 0.9026178010471204, + "fcm_dpo/beta": 0.002479498041793704, + "fcm_dpo/delta": -0.0046107154339551926, + "fcm_dpo/margin": 90.15248107910156, + "fcm_dpo/q_t": 0.447639524936676, + "grad_norm": 50.818077087402344, + "learning_rate": 1.4662207078575684e-08, + "logits/chosen": -0.853556215763092, + "logits/rejected": -0.8220837712287903, + "logps/chosen": -526.3070068359375, + "logps/ref_chosen": -275.7247314453125, + "logps/ref_rejected": -268.91729736328125, + "logps/rejected": -609.6519775390625, + "loss": 4.9115, + "margin_dpo/margin_mean": 90.1524658203125, + "margin_dpo/margin_std": 178.93893432617188, + "step": 431 + }, + { + "KL/chosen_KL_mean": -233.45767211914062, + "KL/mean": -291.3896484375, + "KL/rejected_KL_mean": -349.32171630859375, + "KL/std": 166.04766845703125, + "epoch": 0.9047120418848168, + "fcm_dpo/beta": 0.0024858207907527685, + "fcm_dpo/delta": 0.01790335588157177, + "fcm_dpo/margin": 115.8640365600586, + "fcm_dpo/q_t": 0.4340059757232666, + "grad_norm": 57.34628677368164, + "learning_rate": 1.40507706120426e-08, + "logits/chosen": -0.887722373008728, + "logits/rejected": -0.8687385320663452, + "logps/chosen": -524.8777465820312, + "logps/ref_chosen": -291.42010498046875, + "logps/ref_rejected": -255.48202514648438, + "logps/rejected": -604.8037109375, + "loss": 4.7118, + "margin_dpo/margin_mean": 115.8640365600586, + "margin_dpo/margin_std": 193.5400390625, + "step": 432 + }, + { + "KL/chosen_KL_mean": -264.11077880859375, + "KL/mean": -312.563720703125, + "KL/rejected_KL_mean": -361.0166931152344, + "KL/std": 158.07310485839844, + "epoch": 0.9068062827225131, + "fcm_dpo/beta": 0.002510627033188939, + "fcm_dpo/delta": 0.007871107198297977, + "fcm_dpo/margin": 96.9058609008789, + "fcm_dpo/q_t": 0.44348740577697754, + "grad_norm": 75.00899505615234, + "learning_rate": 1.345198738661285e-08, + "logits/chosen": -0.8320577144622803, + "logits/rejected": -0.8262537717819214, + "logps/chosen": -510.33770751953125, + "logps/ref_chosen": -246.2268829345703, + "logps/ref_rejected": -253.65924072265625, + "logps/rejected": -614.6759033203125, + "loss": 4.8285, + "margin_dpo/margin_mean": 96.9058609008789, + "margin_dpo/margin_std": 178.91293334960938, + "step": 433 + }, + { + "KL/chosen_KL_mean": -274.1569519042969, + "KL/mean": -325.5293884277344, + "KL/rejected_KL_mean": -376.90179443359375, + "KL/std": 164.99172973632812, + "epoch": 0.9089005235602095, + "fcm_dpo/beta": 0.002546996111050248, + "fcm_dpo/delta": 0.02380567416548729, + "fcm_dpo/margin": 102.74482727050781, + "fcm_dpo/q_t": 0.43898072838783264, + "grad_norm": 74.7905044555664, + "learning_rate": 1.2865889513213628e-08, + "logits/chosen": -0.8248652219772339, + "logits/rejected": -0.8263947367668152, + "logps/chosen": -569.6187744140625, + "logps/ref_chosen": -295.4618225097656, + "logps/ref_rejected": -256.2254333496094, + "logps/rejected": -633.127197265625, + "loss": 4.7812, + "margin_dpo/margin_mean": 102.74482727050781, + "margin_dpo/margin_std": 179.2010498046875, + "step": 434 + }, + { + "KL/chosen_KL_mean": -244.2108917236328, + "KL/mean": -307.3104248046875, + "KL/rejected_KL_mean": -370.40997314453125, + "KL/std": 169.79331970214844, + "epoch": 0.9109947643979057, + "fcm_dpo/beta": 0.0025554713793098927, + "fcm_dpo/delta": -0.02224489115178585, + "fcm_dpo/margin": 126.19908142089844, + "fcm_dpo/q_t": 0.42764154076576233, + "grad_norm": 70.11904907226562, + "learning_rate": 1.2292508422495157e-08, + "logits/chosen": -0.8194464445114136, + "logits/rejected": -0.8064465522766113, + "logps/chosen": -504.9492492675781, + "logps/ref_chosen": -260.7384033203125, + "logps/ref_rejected": -248.5688018798828, + "logps/rejected": -618.978759765625, + "loss": 4.6041, + "margin_dpo/margin_mean": 126.19908142089844, + "margin_dpo/margin_std": 186.30056762695312, + "step": 435 + }, + { + "KL/chosen_KL_mean": -273.79022216796875, + "KL/mean": -318.10125732421875, + "KL/rejected_KL_mean": -362.4123840332031, + "KL/std": 188.65408325195312, + "epoch": 0.9130890052356021, + "fcm_dpo/beta": 0.002511697355657816, + "fcm_dpo/delta": -0.003989637829363346, + "fcm_dpo/margin": 88.62214660644531, + "fcm_dpo/q_t": 0.4518389403820038, + "grad_norm": 63.78618240356445, + "learning_rate": 1.1731874863145142e-08, + "logits/chosen": -0.81931072473526, + "logits/rejected": -0.8128557205200195, + "logps/chosen": -593.1127319335938, + "logps/ref_chosen": -319.3224792480469, + "logps/ref_rejected": -299.30322265625, + "logps/rejected": -661.715576171875, + "loss": 4.9787, + "margin_dpo/margin_mean": 88.62214660644531, + "margin_dpo/margin_std": 211.4658966064453, + "step": 436 + }, + { + "KL/chosen_KL_mean": -249.11143493652344, + "KL/mean": -312.220703125, + "KL/rejected_KL_mean": -375.3299865722656, + "KL/std": 183.77027893066406, + "epoch": 0.9151832460732985, + "fcm_dpo/beta": 0.0025260450784116983, + "fcm_dpo/delta": 0.0047508468851447105, + "fcm_dpo/margin": 126.21853637695312, + "fcm_dpo/q_t": 0.42712879180908203, + "grad_norm": 65.6551742553711, + "learning_rate": 1.118401890024001e-08, + "logits/chosen": -0.8493269085884094, + "logits/rejected": -0.8305841684341431, + "logps/chosen": -527.940185546875, + "logps/ref_chosen": -278.82879638671875, + "logps/ref_rejected": -272.55303955078125, + "logps/rejected": -647.8829956054688, + "loss": 4.6504, + "margin_dpo/margin_mean": 126.21853637695312, + "margin_dpo/margin_std": 204.38760375976562, + "step": 437 + }, + { + "KL/chosen_KL_mean": -263.64837646484375, + "KL/mean": -293.3003845214844, + "KL/rejected_KL_mean": -322.9523620605469, + "KL/std": 168.18655395507812, + "epoch": 0.9172774869109948, + "fcm_dpo/beta": 0.0025404624175280333, + "fcm_dpo/delta": 0.007852256298065186, + "fcm_dpo/margin": 59.30400466918945, + "fcm_dpo/q_t": 0.46499937772750854, + "grad_norm": 64.0697021484375, + "learning_rate": 1.06489699136324e-08, + "logits/chosen": -0.8310567736625671, + "logits/rejected": -0.840906023979187, + "logps/chosen": -522.9674682617188, + "logps/ref_chosen": -259.31903076171875, + "logps/ref_rejected": -240.99581909179688, + "logps/rejected": -563.9481811523438, + "loss": 5.2207, + "margin_dpo/margin_mean": 59.303993225097656, + "margin_dpo/margin_std": 191.96351623535156, + "step": 438 + }, + { + "KL/chosen_KL_mean": -252.53065490722656, + "KL/mean": -308.7279357910156, + "KL/rejected_KL_mean": -364.92523193359375, + "KL/std": 159.78070068359375, + "epoch": 0.9193717277486911, + "fcm_dpo/beta": 0.0025289033073931932, + "fcm_dpo/delta": -0.005491324700415134, + "fcm_dpo/margin": 112.39454650878906, + "fcm_dpo/q_t": 0.4330964684486389, + "grad_norm": 69.76829528808594, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": -0.8097501993179321, + "logits/rejected": -0.8053499460220337, + "logps/chosen": -509.65509033203125, + "logps/ref_chosen": -257.1243896484375, + "logps/ref_rejected": -243.20416259765625, + "logps/rejected": -608.1293334960938, + "loss": 4.6923, + "margin_dpo/margin_mean": 112.39454650878906, + "margin_dpo/margin_std": 177.54962158203125, + "step": 439 + }, + { + "KL/chosen_KL_mean": -271.7328186035156, + "KL/mean": -313.9856872558594, + "KL/rejected_KL_mean": -356.2385559082031, + "KL/std": 148.13238525390625, + "epoch": 0.9214659685863874, + "fcm_dpo/beta": 0.0025394840631633997, + "fcm_dpo/delta": 0.01721479743719101, + "fcm_dpo/margin": 84.5057601928711, + "fcm_dpo/q_t": 0.4492788314819336, + "grad_norm": 85.56015014648438, + "learning_rate": 9.617406953185136e-09, + "logits/chosen": -0.8681034445762634, + "logits/rejected": -0.8587910532951355, + "logps/chosen": -579.2643432617188, + "logps/ref_chosen": -307.5315246582031, + "logps/ref_rejected": -264.3540954589844, + "logps/rejected": -620.5927124023438, + "loss": 4.9114, + "margin_dpo/margin_mean": 84.5057601928711, + "margin_dpo/margin_std": 158.16221618652344, + "step": 440 + }, + { + "KL/chosen_KL_mean": -251.61834716796875, + "KL/mean": -314.0332336425781, + "KL/rejected_KL_mean": -376.44805908203125, + "KL/std": 164.15542602539062, + "epoch": 0.9235602094240838, + "fcm_dpo/beta": 0.0025592236779630184, + "fcm_dpo/delta": -0.005991383455693722, + "fcm_dpo/margin": 124.82971954345703, + "fcm_dpo/q_t": 0.425258606672287, + "grad_norm": 64.42757415771484, + "learning_rate": 9.12094829893642e-09, + "logits/chosen": -0.8236322402954102, + "logits/rejected": -0.8174566626548767, + "logps/chosen": -561.6002807617188, + "logps/ref_chosen": -309.9819641113281, + "logps/ref_rejected": -297.4968566894531, + "logps/rejected": -673.9448852539062, + "loss": 4.5766, + "margin_dpo/margin_mean": 124.82971954345703, + "margin_dpo/margin_std": 175.52149963378906, + "step": 441 + }, + { + "KL/chosen_KL_mean": -281.2906494140625, + "KL/mean": -326.7481994628906, + "KL/rejected_KL_mean": -372.2057189941406, + "KL/std": 168.01153564453125, + "epoch": 0.9256544502617801, + "fcm_dpo/beta": 0.002559695625677705, + "fcm_dpo/delta": 0.007131714373826981, + "fcm_dpo/margin": 90.91503143310547, + "fcm_dpo/q_t": 0.44628316164016724, + "grad_norm": 72.68498229980469, + "learning_rate": 8.637407257200496e-09, + "logits/chosen": -0.8905078768730164, + "logits/rejected": -0.8481354713439941, + "logps/chosen": -560.2698364257812, + "logps/ref_chosen": -278.9791564941406, + "logps/ref_rejected": -242.87310791015625, + "logps/rejected": -615.0787963867188, + "loss": 4.9229, + "margin_dpo/margin_mean": 90.91503143310547, + "margin_dpo/margin_std": 197.5365753173828, + "step": 442 + }, + { + "KL/chosen_KL_mean": -262.447021484375, + "KL/mean": -311.3345947265625, + "KL/rejected_KL_mean": -360.22210693359375, + "KL/std": 150.85263061523438, + "epoch": 0.9277486910994764, + "fcm_dpo/beta": 0.002550204750150442, + "fcm_dpo/delta": -0.021942952647805214, + "fcm_dpo/margin": 97.77508544921875, + "fcm_dpo/q_t": 0.44047561287879944, + "grad_norm": 80.34606170654297, + "learning_rate": 8.166809758815895e-09, + "logits/chosen": -0.7915633320808411, + "logits/rejected": -0.8125393390655518, + "logps/chosen": -536.006103515625, + "logps/ref_chosen": -273.5590515136719, + "logps/ref_rejected": -264.0199279785156, + "logps/rejected": -624.2420043945312, + "loss": 4.8309, + "margin_dpo/margin_mean": 97.77508544921875, + "margin_dpo/margin_std": 177.55967712402344, + "step": 443 + }, + { + "KL/chosen_KL_mean": -255.01412963867188, + "KL/mean": -312.3970031738281, + "KL/rejected_KL_mean": -369.7798767089844, + "KL/std": 177.91873168945312, + "epoch": 0.9298429319371728, + "fcm_dpo/beta": 0.002535287756472826, + "fcm_dpo/delta": 0.018113907426595688, + "fcm_dpo/margin": 114.76580047607422, + "fcm_dpo/q_t": 0.43355995416641235, + "grad_norm": 61.830787658691406, + "learning_rate": 7.709181040498253e-09, + "logits/chosen": -0.7957402467727661, + "logits/rejected": -0.7812896370887756, + "logps/chosen": -553.1583251953125, + "logps/ref_chosen": -298.1441955566406, + "logps/ref_rejected": -268.0572814941406, + "logps/rejected": -637.837158203125, + "loss": 4.7328, + "margin_dpo/margin_mean": 114.76579284667969, + "margin_dpo/margin_std": 204.9227294921875, + "step": 444 + }, + { + "KL/chosen_KL_mean": -255.86961364746094, + "KL/mean": -306.81475830078125, + "KL/rejected_KL_mean": -357.7599182128906, + "KL/std": 171.42166137695312, + "epoch": 0.9319371727748691, + "fcm_dpo/beta": 0.0025360870640724897, + "fcm_dpo/delta": -0.026608020067214966, + "fcm_dpo/margin": 101.89030456542969, + "fcm_dpo/q_t": 0.44091877341270447, + "grad_norm": 75.62554168701172, + "learning_rate": 7.2645456434869965e-09, + "logits/chosen": -0.876228392124176, + "logits/rejected": -0.8805335760116577, + "logps/chosen": -510.4102783203125, + "logps/ref_chosen": -254.54067993164062, + "logps/ref_rejected": -264.2445983886719, + "logps/rejected": -622.0044555664062, + "loss": 4.8553, + "margin_dpo/margin_mean": 101.89029693603516, + "margin_dpo/margin_std": 197.63458251953125, + "step": 445 + }, + { + "KL/chosen_KL_mean": -250.0303955078125, + "KL/mean": -306.152587890625, + "KL/rejected_KL_mean": -362.2747497558594, + "KL/std": 161.83197021484375, + "epoch": 0.9340314136125655, + "fcm_dpo/beta": 0.002476347843185067, + "fcm_dpo/delta": -0.010473774746060371, + "fcm_dpo/margin": 112.24434661865234, + "fcm_dpo/q_t": 0.4361517131328583, + "grad_norm": 49.15285110473633, + "learning_rate": 6.832927412229017e-09, + "logits/chosen": -0.7973624467849731, + "logits/rejected": -0.8008688688278198, + "logps/chosen": -556.7528686523438, + "logps/ref_chosen": -306.72247314453125, + "logps/ref_rejected": -266.3735656738281, + "logps/rejected": -628.6482543945312, + "loss": 4.7508, + "margin_dpo/margin_mean": 112.24434661865234, + "margin_dpo/margin_std": 195.67247009277344, + "step": 446 + }, + { + "KL/chosen_KL_mean": -246.97784423828125, + "KL/mean": -308.9893798828125, + "KL/rejected_KL_mean": -371.0009460449219, + "KL/std": 164.9072265625, + "epoch": 0.9361256544502617, + "fcm_dpo/beta": 0.002457038965076208, + "fcm_dpo/delta": -0.005904083605855703, + "fcm_dpo/margin": 124.02310180664062, + "fcm_dpo/q_t": 0.4291154742240906, + "grad_norm": 69.2431640625, + "learning_rate": 6.414349493100129e-09, + "logits/chosen": -0.7988148331642151, + "logits/rejected": -0.7919713258743286, + "logps/chosen": -507.4951171875, + "logps/ref_chosen": -260.51727294921875, + "logps/ref_rejected": -236.47061157226562, + "logps/rejected": -607.4715576171875, + "loss": 4.6405, + "margin_dpo/margin_mean": 124.02310180664062, + "margin_dpo/margin_std": 185.88461303710938, + "step": 447 + }, + { + "KL/chosen_KL_mean": -260.0938415527344, + "KL/mean": -320.5010070800781, + "KL/rejected_KL_mean": -380.908203125, + "KL/std": 160.903076171875, + "epoch": 0.9382198952879581, + "fcm_dpo/beta": 0.0024337496142834425, + "fcm_dpo/delta": -0.020262887701392174, + "fcm_dpo/margin": 120.81436157226562, + "fcm_dpo/q_t": 0.43088439106941223, + "grad_norm": 60.84456253051758, + "learning_rate": 6.0088343331638756e-09, + "logits/chosen": -0.8034505248069763, + "logits/rejected": -0.7938514351844788, + "logps/chosen": -528.880859375, + "logps/ref_chosen": -268.78704833984375, + "logps/ref_rejected": -262.1703796386719, + "logps/rejected": -643.0785522460938, + "loss": 4.6526, + "margin_dpo/margin_mean": 120.81436157226562, + "margin_dpo/margin_std": 174.54934692382812, + "step": 448 + }, + { + "KL/chosen_KL_mean": -247.74948120117188, + "KL/mean": -304.65057373046875, + "KL/rejected_KL_mean": -361.5516052246094, + "KL/std": 163.8621826171875, + "epoch": 0.9403141361256544, + "fcm_dpo/beta": 0.002439489820972085, + "fcm_dpo/delta": 0.02136923372745514, + "fcm_dpo/margin": 113.80213928222656, + "fcm_dpo/q_t": 0.4335501194000244, + "grad_norm": 85.0506362915039, + "learning_rate": 5.616403678967624e-09, + "logits/chosen": -0.9094984531402588, + "logits/rejected": -0.8936970829963684, + "logps/chosen": -578.700927734375, + "logps/ref_chosen": -330.9514465332031, + "logps/ref_rejected": -239.76974487304688, + "logps/rejected": -601.3213500976562, + "loss": 4.76, + "margin_dpo/margin_mean": 113.80213928222656, + "margin_dpo/margin_std": 199.1973114013672, + "step": 449 + }, + { + "KL/chosen_KL_mean": -270.36553955078125, + "KL/mean": -320.8754577636719, + "KL/rejected_KL_mean": -371.3853454589844, + "KL/std": 169.2615203857422, + "epoch": 0.9424083769633508, + "fcm_dpo/beta": 0.0024614077992737293, + "fcm_dpo/delta": -0.004837765824049711, + "fcm_dpo/margin": 101.0197982788086, + "fcm_dpo/q_t": 0.44178810715675354, + "grad_norm": 61.14320373535156, + "learning_rate": 5.2370785753763356e-09, + "logits/chosen": -0.8040182590484619, + "logits/rejected": -0.7969501614570618, + "logps/chosen": -554.6309814453125, + "logps/ref_chosen": -284.26544189453125, + "logps/ref_rejected": -250.5401611328125, + "logps/rejected": -621.925537109375, + "loss": 4.8008, + "margin_dpo/margin_mean": 101.0197982788086, + "margin_dpo/margin_std": 170.01528930664062, + "step": 450 + }, + { + "KL/chosen_KL_mean": -279.3070068359375, + "KL/mean": -320.45257568359375, + "KL/rejected_KL_mean": -361.5981140136719, + "KL/std": 158.63836669921875, + "epoch": 0.9445026178010472, + "fcm_dpo/beta": 0.0024547951761633158, + "fcm_dpo/delta": 0.0002841167151927948, + "fcm_dpo/margin": 82.29108428955078, + "fcm_dpo/q_t": 0.4520818889141083, + "grad_norm": 81.18460845947266, + "learning_rate": 4.8708793644441086e-09, + "logits/chosen": -0.8152300715446472, + "logits/rejected": -0.7828786373138428, + "logps/chosen": -581.6279296875, + "logps/ref_chosen": -302.3209228515625, + "logps/ref_rejected": -254.09747314453125, + "logps/rejected": -615.6956176757812, + "loss": 4.9711, + "margin_dpo/margin_mean": 82.29108428955078, + "margin_dpo/margin_std": 171.24998474121094, + "step": 451 + }, + { + "KL/chosen_KL_mean": -255.58740234375, + "KL/mean": -323.3721008300781, + "KL/rejected_KL_mean": -391.1568603515625, + "KL/std": 196.24913024902344, + "epoch": 0.9465968586387434, + "fcm_dpo/beta": 0.002398530952632427, + "fcm_dpo/delta": -0.03113538958132267, + "fcm_dpo/margin": 135.5694580078125, + "fcm_dpo/q_t": 0.42863401770591736, + "grad_norm": 50.24382781982422, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": -0.8734545707702637, + "logits/rejected": -0.846143364906311, + "logps/chosen": -554.9795532226562, + "logps/ref_chosen": -299.39215087890625, + "logps/ref_rejected": -284.3475036621094, + "logps/rejected": -675.50439453125, + "loss": 4.6565, + "margin_dpo/margin_mean": 135.5694580078125, + "margin_dpo/margin_std": 229.8317108154297, + "step": 452 + }, + { + "KL/chosen_KL_mean": -275.34716796875, + "KL/mean": -332.12860107421875, + "KL/rejected_KL_mean": -388.9101257324219, + "KL/std": 167.65248107910156, + "epoch": 0.9486910994764398, + "fcm_dpo/beta": 0.0024174817372113466, + "fcm_dpo/delta": 0.01900402083992958, + "fcm_dpo/margin": 113.56294250488281, + "fcm_dpo/q_t": 0.4362477958202362, + "grad_norm": 65.24600982666016, + "learning_rate": 4.1779364682113794e-09, + "logits/chosen": -0.8175550699234009, + "logits/rejected": -0.8079196214675903, + "logps/chosen": -599.9988403320312, + "logps/ref_chosen": -324.6517028808594, + "logps/ref_rejected": -304.1527099609375, + "logps/rejected": -693.0628051757812, + "loss": 4.7582, + "margin_dpo/margin_mean": 113.56295013427734, + "margin_dpo/margin_std": 204.01402282714844, + "step": 453 + }, + { + "KL/chosen_KL_mean": -261.29437255859375, + "KL/mean": -329.093994140625, + "KL/rejected_KL_mean": -396.8935546875, + "KL/std": 192.02064514160156, + "epoch": 0.9507853403141361, + "fcm_dpo/beta": 0.002420138567686081, + "fcm_dpo/delta": -0.012020056135952473, + "fcm_dpo/margin": 135.59918212890625, + "fcm_dpo/q_t": 0.42454957962036133, + "grad_norm": 73.93843841552734, + "learning_rate": 3.851229943335393e-09, + "logits/chosen": -0.8420263528823853, + "logits/rejected": -0.8489396572113037, + "logps/chosen": -560.9061279296875, + "logps/ref_chosen": -299.6117248535156, + "logps/ref_rejected": -303.74224853515625, + "logps/rejected": -700.6358032226562, + "loss": 4.6019, + "margin_dpo/margin_mean": 135.59918212890625, + "margin_dpo/margin_std": 211.62002563476562, + "step": 454 + }, + { + "KL/chosen_KL_mean": -254.47279357910156, + "KL/mean": -297.5650939941406, + "KL/rejected_KL_mean": -340.65740966796875, + "KL/std": 164.42532348632812, + "epoch": 0.9528795811518325, + "fcm_dpo/beta": 0.002410230925306678, + "fcm_dpo/delta": -0.0064119016751646996, + "fcm_dpo/margin": 86.18463897705078, + "fcm_dpo/q_t": 0.45259642601013184, + "grad_norm": 60.235843658447266, + "learning_rate": 3.5377236299748147e-09, + "logits/chosen": -0.8099350333213806, + "logits/rejected": -0.8183305263519287, + "logps/chosen": -528.08447265625, + "logps/ref_chosen": -273.6116943359375, + "logps/ref_rejected": -274.4293518066406, + "logps/rejected": -615.0867919921875, + "loss": 4.9766, + "margin_dpo/margin_mean": 86.18463897705078, + "margin_dpo/margin_std": 188.7678985595703, + "step": 455 + }, + { + "KL/chosen_KL_mean": -284.4190979003906, + "KL/mean": -342.2475280761719, + "KL/rejected_KL_mean": -400.07598876953125, + "KL/std": 191.12667846679688, + "epoch": 0.9549738219895288, + "fcm_dpo/beta": 0.0024038502015173435, + "fcm_dpo/delta": -0.0050141820684075356, + "fcm_dpo/margin": 115.6568374633789, + "fcm_dpo/q_t": 0.4384552836418152, + "grad_norm": 72.66372680664062, + "learning_rate": 3.2374343405217884e-09, + "logits/chosen": -0.7380684614181519, + "logits/rejected": -0.7515442967414856, + "logps/chosen": -606.591064453125, + "logps/ref_chosen": -322.17193603515625, + "logps/ref_rejected": -294.54461669921875, + "logps/rejected": -694.6205444335938, + "loss": 4.7994, + "margin_dpo/margin_mean": 115.6568374633789, + "margin_dpo/margin_std": 224.9918212890625, + "step": 456 + }, + { + "KL/chosen_KL_mean": -275.81463623046875, + "KL/mean": -339.070068359375, + "KL/rejected_KL_mean": -402.3255615234375, + "KL/std": 183.56442260742188, + "epoch": 0.9570680628272251, + "fcm_dpo/beta": 0.0023992531932890415, + "fcm_dpo/delta": 0.008432567119598389, + "fcm_dpo/margin": 126.51089477539062, + "fcm_dpo/q_t": 0.4272172451019287, + "grad_norm": 64.16017150878906, + "learning_rate": 2.9503781785795713e-09, + "logits/chosen": -0.8137893080711365, + "logits/rejected": -0.8142549991607666, + "logps/chosen": -583.6109008789062, + "logps/ref_chosen": -307.7962341308594, + "logps/ref_rejected": -274.5501403808594, + "logps/rejected": -676.8756713867188, + "loss": 4.6952, + "margin_dpo/margin_mean": 126.51089477539062, + "margin_dpo/margin_std": 209.9931182861328, + "step": 457 + }, + { + "KL/chosen_KL_mean": -270.3670349121094, + "KL/mean": -323.4082946777344, + "KL/rejected_KL_mean": -376.44952392578125, + "KL/std": 184.91030883789062, + "epoch": 0.9591623036649215, + "fcm_dpo/beta": 0.002431598724797368, + "fcm_dpo/delta": 0.017081839963793755, + "fcm_dpo/margin": 106.08248901367188, + "fcm_dpo/q_t": 0.4409106373786926, + "grad_norm": 64.75917053222656, + "learning_rate": 2.6765705380989432e-09, + "logits/chosen": -0.8364468812942505, + "logits/rejected": -0.8125419020652771, + "logps/chosen": -567.398681640625, + "logps/ref_chosen": -297.0316467285156, + "logps/ref_rejected": -276.1112365722656, + "logps/rejected": -652.560791015625, + "loss": 4.8427, + "margin_dpo/margin_mean": 106.08248901367188, + "margin_dpo/margin_std": 206.88600158691406, + "step": 458 + }, + { + "KL/chosen_KL_mean": -255.69676208496094, + "KL/mean": -301.064697265625, + "KL/rejected_KL_mean": -346.4325866699219, + "KL/std": 156.7518768310547, + "epoch": 0.9612565445026178, + "fcm_dpo/beta": 0.002469552680850029, + "fcm_dpo/delta": 0.03057432547211647, + "fcm_dpo/margin": 90.73584747314453, + "fcm_dpo/q_t": 0.44875121116638184, + "grad_norm": 76.80902862548828, + "learning_rate": 2.416026102552732e-09, + "logits/chosen": -0.8981096148490906, + "logits/rejected": -0.8896926641464233, + "logps/chosen": -549.2220458984375, + "logps/ref_chosen": -293.5252990722656, + "logps/ref_rejected": -289.30126953125, + "logps/rejected": -635.7338256835938, + "loss": 4.9246, + "margin_dpo/margin_mean": 90.73584747314453, + "margin_dpo/margin_std": 187.47048950195312, + "step": 459 + }, + { + "KL/chosen_KL_mean": -263.8416442871094, + "KL/mean": -310.0146789550781, + "KL/rejected_KL_mean": -356.18768310546875, + "KL/std": 150.35638427734375, + "epoch": 0.9633507853403142, + "fcm_dpo/beta": 0.0025106584653258324, + "fcm_dpo/delta": 0.018712403252720833, + "fcm_dpo/margin": 92.34601593017578, + "fcm_dpo/q_t": 0.44546929001808167, + "grad_norm": 55.63397979736328, + "learning_rate": 2.168758844148272e-09, + "logits/chosen": -0.8555883169174194, + "logits/rejected": -0.8532405495643616, + "logps/chosen": -582.6220092773438, + "logps/ref_chosen": -318.7803649902344, + "logps/ref_rejected": -258.7906799316406, + "logps/rejected": -614.9783325195312, + "loss": 4.8808, + "margin_dpo/margin_mean": 92.34600830078125, + "margin_dpo/margin_std": 182.1141357421875, + "step": 460 + }, + { + "KL/chosen_KL_mean": -259.0890197753906, + "KL/mean": -315.6316833496094, + "KL/rejected_KL_mean": -372.1743469238281, + "KL/std": 172.29824829101562, + "epoch": 0.9654450261780104, + "fcm_dpo/beta": 0.002565775066614151, + "fcm_dpo/delta": 0.022872814908623695, + "fcm_dpo/margin": 113.08531188964844, + "fcm_dpo/q_t": 0.4341578781604767, + "grad_norm": 60.84050750732422, + "learning_rate": 1.9347820230782295e-09, + "logits/chosen": -0.81836998462677, + "logits/rejected": -0.8396799564361572, + "logps/chosen": -502.9990234375, + "logps/ref_chosen": -243.9099884033203, + "logps/ref_rejected": -232.6382293701172, + "logps/rejected": -604.8125610351562, + "loss": 4.7443, + "margin_dpo/margin_mean": 113.08531188964844, + "margin_dpo/margin_std": 204.30162048339844, + "step": 461 + }, + { + "KL/chosen_KL_mean": -262.39306640625, + "KL/mean": -323.4773864746094, + "KL/rejected_KL_mean": -384.56170654296875, + "KL/std": 163.00491333007812, + "epoch": 0.9675392670157068, + "fcm_dpo/beta": 0.002616667188704014, + "fcm_dpo/delta": 0.022976616397500038, + "fcm_dpo/margin": 122.16867065429688, + "fcm_dpo/q_t": 0.4264695942401886, + "grad_norm": 67.28462219238281, + "learning_rate": 1.7141081868094209e-09, + "logits/chosen": -0.8513904213905334, + "logits/rejected": -0.7988516688346863, + "logps/chosen": -606.4840698242188, + "logps/ref_chosen": -344.09100341796875, + "logps/ref_rejected": -252.45037841796875, + "logps/rejected": -637.0120849609375, + "loss": 4.6085, + "margin_dpo/margin_mean": 122.16866302490234, + "margin_dpo/margin_std": 185.75375366210938, + "step": 462 + }, + { + "KL/chosen_KL_mean": -260.8435974121094, + "KL/mean": -307.8876037597656, + "KL/rejected_KL_mean": -354.93157958984375, + "KL/std": 176.79953002929688, + "epoch": 0.9696335078534032, + "fcm_dpo/beta": 0.002635349053889513, + "fcm_dpo/delta": 0.00047087250277400017, + "fcm_dpo/margin": 94.08798217773438, + "fcm_dpo/q_t": 0.4436490535736084, + "grad_norm": 87.89910888671875, + "learning_rate": 1.5067491694100153e-09, + "logits/chosen": -0.8419432640075684, + "logits/rejected": -0.7984883189201355, + "logps/chosen": -557.986083984375, + "logps/ref_chosen": -297.1424560546875, + "logps/ref_rejected": -234.0208282470703, + "logps/rejected": -588.9524536132812, + "loss": 4.9332, + "margin_dpo/margin_mean": 94.08797454833984, + "margin_dpo/margin_std": 211.70285034179688, + "step": 463 + }, + { + "KL/chosen_KL_mean": -269.11505126953125, + "KL/mean": -319.7918701171875, + "KL/rejected_KL_mean": -370.46868896484375, + "KL/std": 167.46054077148438, + "epoch": 0.9717277486910995, + "fcm_dpo/beta": 0.0026447109412401915, + "fcm_dpo/delta": 0.0007200678810477257, + "fcm_dpo/margin": 101.35370635986328, + "fcm_dpo/q_t": 0.43749621510505676, + "grad_norm": 58.30624008178711, + "learning_rate": 1.3127160909147672e-09, + "logits/chosen": -0.8257731199264526, + "logits/rejected": -0.8404132723808289, + "logps/chosen": -534.8258056640625, + "logps/ref_chosen": -265.71075439453125, + "logps/ref_rejected": -256.4108581542969, + "logps/rejected": -626.8795776367188, + "loss": 4.7914, + "margin_dpo/margin_mean": 101.35370635986328, + "margin_dpo/margin_std": 186.4237823486328, + "step": 464 + }, + { + "KL/chosen_KL_mean": -246.98736572265625, + "KL/mean": -310.69110107421875, + "KL/rejected_KL_mean": -374.39483642578125, + "KL/std": 160.25502014160156, + "epoch": 0.9738219895287958, + "fcm_dpo/beta": 0.002568161580711603, + "fcm_dpo/delta": -0.03626430779695511, + "fcm_dpo/margin": 127.40748596191406, + "fcm_dpo/q_t": 0.42483144998550415, + "grad_norm": 69.23091888427734, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": -0.8961881995201111, + "logits/rejected": -0.8565638065338135, + "logps/chosen": -540.1400756835938, + "logps/ref_chosen": -293.1527404785156, + "logps/ref_rejected": -293.70947265625, + "logps/rejected": -668.1043701171875, + "loss": 4.5974, + "margin_dpo/margin_mean": 127.40748596191406, + "margin_dpo/margin_std": 186.34690856933594, + "step": 465 + }, + { + "KL/chosen_KL_mean": -234.4508819580078, + "KL/mean": -300.031982421875, + "KL/rejected_KL_mean": -365.6130676269531, + "KL/std": 157.27207946777344, + "epoch": 0.9759162303664921, + "fcm_dpo/beta": 0.0025337152183055878, + "fcm_dpo/delta": -0.0182378850877285, + "fcm_dpo/margin": 131.1621551513672, + "fcm_dpo/q_t": 0.4229571223258972, + "grad_norm": 57.641929626464844, + "learning_rate": 9.64668657069706e-10, + "logits/chosen": -0.8110611438751221, + "logits/rejected": -0.7530339956283569, + "logps/chosen": -495.928466796875, + "logps/ref_chosen": -261.4775695800781, + "logps/ref_rejected": -248.36282348632812, + "logps/rejected": -613.9758911132812, + "loss": 4.5442, + "margin_dpo/margin_mean": 131.1621551513672, + "margin_dpo/margin_std": 181.0853271484375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -262.05810546875, + "KL/mean": -312.107421875, + "KL/rejected_KL_mean": -362.1567687988281, + "KL/std": 167.3401641845703, + "epoch": 0.9780104712041885, + "fcm_dpo/beta": 0.0024951626546680927, + "fcm_dpo/delta": -0.015738535672426224, + "fcm_dpo/margin": 100.09864807128906, + "fcm_dpo/q_t": 0.4429374635219574, + "grad_norm": 75.20722961425781, + "learning_rate": 8.106729664475176e-10, + "logits/chosen": -0.8004065752029419, + "logits/rejected": -0.7925856709480286, + "logps/chosen": -528.412353515625, + "logps/ref_chosen": -266.354248046875, + "logps/ref_rejected": -277.76324462890625, + "logps/rejected": -639.9200439453125, + "loss": 4.8735, + "margin_dpo/margin_mean": 100.09864807128906, + "margin_dpo/margin_std": 204.54727172851562, + "step": 467 + }, + { + "KL/chosen_KL_mean": -262.4240417480469, + "KL/mean": -304.2724914550781, + "KL/rejected_KL_mean": -346.12091064453125, + "KL/std": 160.87689208984375, + "epoch": 0.9801047120418848, + "fcm_dpo/beta": 0.0025261654518544674, + "fcm_dpo/delta": 0.0298687182366848, + "fcm_dpo/margin": 83.69690704345703, + "fcm_dpo/q_t": 0.4495968222618103, + "grad_norm": 70.44644165039062, + "learning_rate": 6.700405431837585e-10, + "logits/chosen": -0.8833749294281006, + "logits/rejected": -0.8521139025688171, + "logps/chosen": -580.38720703125, + "logps/ref_chosen": -317.9631652832031, + "logps/ref_rejected": -261.8744201660156, + "logps/rejected": -607.9953002929688, + "loss": 5.0089, + "margin_dpo/margin_mean": 83.69690704345703, + "margin_dpo/margin_std": 198.9070281982422, + "step": 468 + }, + { + "KL/chosen_KL_mean": -246.26190185546875, + "KL/mean": -308.28466796875, + "KL/rejected_KL_mean": -370.3074951171875, + "KL/std": 168.65597534179688, + "epoch": 0.9821989528795811, + "fcm_dpo/beta": 0.0025742819998413324, + "fcm_dpo/delta": 0.003947127610445023, + "fcm_dpo/margin": 124.04557800292969, + "fcm_dpo/q_t": 0.42625945806503296, + "grad_norm": 52.03805923461914, + "learning_rate": 5.427789289685347e-10, + "logits/chosen": -0.8123364448547363, + "logits/rejected": -0.795385479927063, + "logps/chosen": -571.148681640625, + "logps/ref_chosen": -324.8868103027344, + "logps/ref_rejected": -264.0421447753906, + "logps/rejected": -634.349609375, + "loss": 4.6346, + "margin_dpo/margin_mean": 124.04557800292969, + "margin_dpo/margin_std": 199.66555786132812, + "step": 469 + }, + { + "KL/chosen_KL_mean": -248.92559814453125, + "KL/mean": -309.4586181640625, + "KL/rejected_KL_mean": -369.9916076660156, + "KL/std": 166.44065856933594, + "epoch": 0.9842931937172775, + "fcm_dpo/beta": 0.0025518168695271015, + "fcm_dpo/delta": -0.006509192753583193, + "fcm_dpo/margin": 121.06607818603516, + "fcm_dpo/q_t": 0.4286976456642151, + "grad_norm": 82.49861145019531, + "learning_rate": 4.288949484559934e-10, + "logits/chosen": -0.8041696548461914, + "logits/rejected": -0.8020331263542175, + "logps/chosen": -563.6297607421875, + "logps/ref_chosen": -314.7042541503906, + "logps/ref_rejected": -259.2276611328125, + "logps/rejected": -629.2192993164062, + "loss": 4.6557, + "margin_dpo/margin_mean": 121.06607818603516, + "margin_dpo/margin_std": 191.92596435546875, + "step": 470 + }, + { + "KL/chosen_KL_mean": -265.6898498535156, + "KL/mean": -325.8057556152344, + "KL/rejected_KL_mean": -385.921630859375, + "KL/std": 194.99688720703125, + "epoch": 0.9863874345549738, + "fcm_dpo/beta": 0.0025341196451336145, + "fcm_dpo/delta": -0.01957012340426445, + "fcm_dpo/margin": 120.23179626464844, + "fcm_dpo/q_t": 0.43427878618240356, + "grad_norm": 76.36981201171875, + "learning_rate": 3.2839470889836627e-10, + "logits/chosen": -0.8662016987800598, + "logits/rejected": -0.8508539199829102, + "logps/chosen": -558.2646484375, + "logps/ref_chosen": -292.5748291015625, + "logps/ref_rejected": -298.7525329589844, + "logps/rejected": -684.6741333007812, + "loss": 4.7386, + "margin_dpo/margin_mean": 120.23179626464844, + "margin_dpo/margin_std": 221.61729431152344, + "step": 471 + }, + { + "KL/chosen_KL_mean": -239.04766845703125, + "KL/mean": -306.37890625, + "KL/rejected_KL_mean": -373.710205078125, + "KL/std": 166.82215881347656, + "epoch": 0.9884816753926702, + "fcm_dpo/beta": 0.0024978769943118095, + "fcm_dpo/delta": 0.0006190729327499866, + "fcm_dpo/margin": 134.66250610351562, + "fcm_dpo/q_t": 0.4230276346206665, + "grad_norm": 88.5049057006836, + "learning_rate": 2.412835998185092e-10, + "logits/chosen": -0.8472277522087097, + "logits/rejected": -0.8603392839431763, + "logps/chosen": -482.4215087890625, + "logps/ref_chosen": -243.37380981445312, + "logps/ref_rejected": -251.12109375, + "logps/rejected": -624.831298828125, + "loss": 4.5638, + "margin_dpo/margin_mean": 134.66250610351562, + "margin_dpo/margin_std": 198.72836303710938, + "step": 472 + }, + { + "KL/chosen_KL_mean": -256.0351867675781, + "KL/mean": -317.4332275390625, + "KL/rejected_KL_mean": -378.8313293457031, + "KL/std": 152.7645263671875, + "epoch": 0.9905759162303664, + "fcm_dpo/beta": 0.0025325005408376455, + "fcm_dpo/delta": 0.027150386944413185, + "fcm_dpo/margin": 122.79612731933594, + "fcm_dpo/q_t": 0.4268743097782135, + "grad_norm": 49.66973114013672, + "learning_rate": 1.6756629272085544e-10, + "logits/chosen": -0.7995487451553345, + "logits/rejected": -0.7962360978126526, + "logps/chosen": -542.36376953125, + "logps/ref_chosen": -286.3286437988281, + "logps/ref_rejected": -258.6535339355469, + "logps/rejected": -637.4849243164062, + "loss": 4.5916, + "margin_dpo/margin_mean": 122.79612731933594, + "margin_dpo/margin_std": 171.07659912109375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -259.002685546875, + "KL/mean": -313.4521179199219, + "KL/rejected_KL_mean": -367.90155029296875, + "KL/std": 170.04241943359375, + "epoch": 0.9926701570680628, + "fcm_dpo/beta": 0.002547986339777708, + "fcm_dpo/delta": -0.010679654777050018, + "fcm_dpo/margin": 108.89888000488281, + "fcm_dpo/q_t": 0.4356432557106018, + "grad_norm": 75.55198669433594, + "learning_rate": 1.072467408408384e-10, + "logits/chosen": -0.8527939319610596, + "logits/rejected": -0.8474220633506775, + "logps/chosen": -547.0923461914062, + "logps/ref_chosen": -288.08966064453125, + "logps/ref_rejected": -266.69696044921875, + "logps/rejected": -634.5985717773438, + "loss": 4.7386, + "margin_dpo/margin_mean": 108.89888763427734, + "margin_dpo/margin_std": 182.4081573486328, + "step": 474 + }, + { + "KL/chosen_KL_mean": -241.1262969970703, + "KL/mean": -290.3180236816406, + "KL/rejected_KL_mean": -339.50970458984375, + "KL/std": 152.7301483154297, + "epoch": 0.9947643979057592, + "fcm_dpo/beta": 0.0025214108172804117, + "fcm_dpo/delta": -0.01645851507782936, + "fcm_dpo/margin": 98.38336181640625, + "fcm_dpo/q_t": 0.4433209002017975, + "grad_norm": 57.4051628112793, + "learning_rate": 6.032817893297793e-11, + "logits/chosen": -0.8317741751670837, + "logits/rejected": -0.84046870470047, + "logps/chosen": -497.1293640136719, + "logps/ref_chosen": -256.0030517578125, + "logps/ref_rejected": -244.50660705566406, + "logps/rejected": -584.0162963867188, + "loss": 4.824, + "margin_dpo/margin_mean": 98.38336181640625, + "margin_dpo/margin_std": 176.57955932617188, + "step": 475 + }, + { + "KL/chosen_KL_mean": -242.7122039794922, + "KL/mean": -295.69720458984375, + "KL/rejected_KL_mean": -348.68218994140625, + "KL/std": 153.21731567382812, + "epoch": 0.9968586387434555, + "fcm_dpo/beta": 0.0024653631262481213, + "fcm_dpo/delta": -0.029824629426002502, + "fcm_dpo/margin": 105.96993255615234, + "fcm_dpo/q_t": 0.44073987007141113, + "grad_norm": 54.92226028442383, + "learning_rate": 2.6813123097352287e-11, + "logits/chosen": -0.8974910378456116, + "logits/rejected": -0.8506391048431396, + "logps/chosen": -564.1797485351562, + "logps/ref_chosen": -321.467529296875, + "logps/ref_rejected": -295.0592956542969, + "logps/rejected": -643.7415161132812, + "loss": 4.8239, + "margin_dpo/margin_mean": 105.96994018554688, + "margin_dpo/margin_std": 196.14195251464844, + "step": 476 + }, + { + "KL/chosen_KL_mean": -268.98797607421875, + "KL/mean": -318.58404541015625, + "KL/rejected_KL_mean": -368.18011474609375, + "KL/std": 177.54359436035156, + "epoch": 0.9989528795811519, + "fcm_dpo/beta": 0.002445072866976261, + "fcm_dpo/delta": -0.009669403545558453, + "fcm_dpo/margin": 99.1921615600586, + "fcm_dpo/q_t": 0.44362398982048035, + "grad_norm": 69.17218017578125, + "learning_rate": 6.7033706447061635e-12, + "logits/chosen": -0.7706684470176697, + "logits/rejected": -0.7763949632644653, + "logps/chosen": -545.7819213867188, + "logps/ref_chosen": -276.7939758300781, + "logps/ref_rejected": -244.82919311523438, + "logps/rejected": -613.00927734375, + "loss": 4.9399, + "margin_dpo/margin_mean": 99.19215393066406, + "margin_dpo/margin_std": 223.55364990234375, + "step": 477 + }, + { + "epoch": 0.9989528795811519, + "step": 477, + "total_flos": 0.0, + "train_loss": 4.88172167052263, + "train_runtime": 7274.147, + "train_samples_per_second": 8.404, + "train_steps_per_second": 0.066 + } + ], + "logging_steps": 1, + "max_steps": 477, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..5969111 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb01fe9789fb9e948a5f7aad6836f3d3f360ff68497aa2ceb13dbb9cfeb4515c +size 8312