From b724221a776240f544064da9a9ca5920c2766b76 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 14 May 2026 14:57:50 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 Source: Original Platform --- .gitattributes | 36 + README.md | 62 + all_results.json | 9 + config.json | 29 + generation_config.json | 9 + model-00001-of-00007.safetensors | 3 + model-00002-of-00007.safetensors | 3 + model-00003-of-00007.safetensors | 3 + model-00004-of-00007.safetensors | 3 + model-00005-of-00007.safetensors | 3 + model-00006-of-00007.safetensors | 3 + model-00007-of-00007.safetensors | 3 + model.safetensors.index.json | 298 + special_tokens_map.json | 23 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++ train_results.json | 9 + trainer_state.json | 15706 +++++++++++++++++++++++++++++ 18 files changed, 18269 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00007.safetensors create mode 100644 model-00002-of-00007.safetensors create mode 100644 model-00003-of-00007.safetensors create mode 100644 model-00004-of-00007.safetensors create mode 100644 model-00005-of-00007.safetensors create mode 100644 model-00006-of-00007.safetensors create mode 100644 model-00007-of-00007.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..581b660 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.5 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..27d2fc6 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0915097299301326, + "train_runtime": 1740.2795, + "train_samples": 43598, + "train_samples_per_second": 25.052, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..1edebd4 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98858434d0dfe66a44638b4dc4b1ba41764be3b923b1c9c7bb1a4e2ed526606c +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..c3f6c48 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffa45e9e360096fd8d3b3893e22e8671b883f2cbf6a0c90bcdceed1f1fac5ebc +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..5af4a5d --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83d3c89e4a2891414eaca091030ade151ac1eeb809925abf53fc9a91dd287bb3 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..89b9826 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e2d0a3bd34a7709cf91c9d91863be243cab9b153c3980be09cdb33b3848a4cc +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..73cfc28 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd9a75e1dcb88a444ed1e444dfcd806035e56232ba26724eabaceb08bfa00dd +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..a31c5dd --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faeb4668a18825e4ec543a568bd7e81421b135f8cfca9055a2f78437ee3addd3 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..21262ef --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b89255f9d0db146153994312a7e9da705101afb0c0aea3bbea65e36821bf0f +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..27d2fc6 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0915097299301326, + "train_runtime": 1740.2795, + "train_samples": 43598, + "train_samples_per_second": 25.052, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..27952dc --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.1020384356379509, + "fcm_dpo/delta": 0.19979780912399292, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 85.2863540649414, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10407686978578568, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.501709520816803, + "grad_norm": 75.19337463378906, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3935, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.012233734130859375, + "KL/mean": 0.013328447937965393, + "KL/rejected_KL_mean": 0.014421463012695312, + "KL/std": 0.23720939457416534, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.106195367872715, + "fcm_dpo/delta": 0.19951710104942322, + "fcm_dpo/margin": -0.0021870434284210205, + "fcm_dpo/q_t": 0.50005704164505, + "grad_norm": 75.44918823242188, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.48161962628364563, + "logits/rejected": -0.44215089082717896, + "logps/chosen": -60.9693603515625, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.65817260742188, + "loss": 1.3868, + "margin_dpo/margin_mean": -0.002186805009841919, + "margin_dpo/margin_std": 0.33678072690963745, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.013614654541015625, + "KL/mean": 0.04838371276855469, + "KL/rejected_KL_mean": 0.08315658569335938, + "KL/std": 0.25272637605667114, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10831386595964432, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06953638792037964, + "fcm_dpo/q_t": 0.5018829107284546, + "grad_norm": 78.01704406738281, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.4681774973869324, + "logits/rejected": -0.4405352473258972, + "logps/chosen": -56.75409698486328, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.56394958496094, + "loss": 1.3943, + "margin_dpo/margin_mean": -0.06953713297843933, + "margin_dpo/margin_std": 0.3787645101547241, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.015838623046875, + "KL/mean": 0.006019815802574158, + "KL/rejected_KL_mean": -0.0037994384765625, + "KL/std": 0.26149606704711914, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.11499126255512238, + "fcm_dpo/delta": 0.39785218238830566, + "fcm_dpo/margin": 0.01963701844215393, + "fcm_dpo/q_t": 0.4994630813598633, + "grad_norm": 102.74085235595703, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5144762396812439, + "logits/rejected": -0.4707754850387573, + "logps/chosen": -53.843536376953125, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.15298461914062, + "loss": 1.3845, + "margin_dpo/margin_mean": 0.019636929035186768, + "margin_dpo/margin_std": 0.3583635687828064, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.005222320556640625, + "KL/mean": 0.0015716701745986938, + "KL/rejected_KL_mean": 0.008365631103515625, + "KL/std": 0.24871157109737396, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.1172846183180809, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.013584703207015991, + "fcm_dpo/q_t": 0.5003972053527832, + "grad_norm": 107.95716094970703, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.510610818862915, + "logits/rejected": -0.4688519239425659, + "logps/chosen": -63.0127067565918, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.63697814941406, + "loss": 1.3883, + "margin_dpo/margin_mean": -0.013584673404693604, + "margin_dpo/margin_std": 0.3626948297023773, + "step": 6 + }, + { + "KL/chosen_KL_mean": -0.005901336669921875, + "KL/mean": 0.013675212860107422, + "KL/rejected_KL_mean": 0.0332489013671875, + "KL/std": 0.3157729506492615, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.1172846183180809, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03915220499038696, + "fcm_dpo/q_t": 0.5011464357376099, + "grad_norm": 96.8883056640625, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.496779203414917, + "logits/rejected": -0.46308404207229614, + "logps/chosen": -57.780723571777344, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.88734436035156, + "loss": 1.3915, + "margin_dpo/margin_mean": -0.039151787757873535, + "margin_dpo/margin_std": 0.4199862480163574, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.03875541687011719, + "KL/mean": 0.058960139751434326, + "KL/rejected_KL_mean": 0.07917022705078125, + "KL/std": 0.2766719460487366, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.12204919755458832, + "fcm_dpo/delta": 0.1991029679775238, + "fcm_dpo/margin": -0.04041558504104614, + "fcm_dpo/q_t": 0.501240611076355, + "grad_norm": 96.34102630615234, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5006585121154785, + "logits/rejected": -0.47387760877609253, + "logps/chosen": -58.67728042602539, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.23225402832031, + "loss": 1.3918, + "margin_dpo/margin_mean": -0.04041558504104614, + "margin_dpo/margin_std": 0.3688473105430603, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.010095596313476562, + "KL/mean": 0.008026927709579468, + "KL/rejected_KL_mean": 0.026153564453125, + "KL/std": 0.28498250246047974, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.12204919755458832, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03624141216278076, + "fcm_dpo/q_t": 0.5011049509048462, + "grad_norm": 104.20061492919922, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.48470985889434814, + "logits/rejected": -0.43760398030281067, + "logps/chosen": -69.87693786621094, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.57650756835938, + "loss": 1.3914, + "margin_dpo/margin_mean": -0.03624224662780762, + "margin_dpo/margin_std": 0.4175499677658081, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.014812469482421875, + "KL/mean": 0.026996105909347534, + "KL/rejected_KL_mean": 0.0391845703125, + "KL/std": 0.2730364203453064, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.12452996522188187, + "fcm_dpo/delta": 0.1992366760969162, + "fcm_dpo/margin": -0.024369359016418457, + "fcm_dpo/q_t": 0.5007413029670715, + "grad_norm": 88.73687744140625, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.4830132722854614, + "logits/rejected": -0.4392854869365692, + "logps/chosen": -48.342872619628906, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.33287811279297, + "loss": 1.3898, + "margin_dpo/margin_mean": -0.02436944842338562, + "margin_dpo/margin_std": 0.3846496343612671, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.028707504272460938, + "KL/mean": 0.016712307929992676, + "KL/rejected_KL_mean": 0.0047149658203125, + "KL/std": 0.24994134902954102, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.1320638358592987, + "fcm_dpo/delta": 0.1950692981481552, + "fcm_dpo/margin": 0.023987650871276855, + "fcm_dpo/q_t": 0.4992583394050598, + "grad_norm": 90.11399841308594, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.46979549527168274, + "logits/rejected": -0.44525203108787537, + "logps/chosen": -52.9881477355957, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.77566528320312, + "loss": 1.3837, + "margin_dpo/margin_mean": 0.02398782968521118, + "margin_dpo/margin_std": 0.35764360427856445, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.012472152709960938, + "KL/mean": -0.011578842997550964, + "KL/rejected_KL_mean": -0.01068115234375, + "KL/std": 0.27221232652664185, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.1346975713968277, + "fcm_dpo/delta": 0.19555437564849854, + "fcm_dpo/margin": -0.0017872154712677002, + "fcm_dpo/q_t": 0.5000562071800232, + "grad_norm": 125.7822494506836, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5281082391738892, + "logits/rejected": -0.4917023777961731, + "logps/chosen": -61.817909240722656, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.86895751953125, + "loss": 1.3871, + "margin_dpo/margin_mean": -0.001787722110748291, + "margin_dpo/margin_std": 0.38611578941345215, + "step": 12 + }, + { + "KL/chosen_KL_mean": -0.0322723388671875, + "KL/mean": -0.0176934152841568, + "KL/rejected_KL_mean": -0.00311279296875, + "KL/std": 0.3298068046569824, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.14007024466991425, + "fcm_dpo/delta": 0.1955653429031372, + "fcm_dpo/margin": -0.02915528416633606, + "fcm_dpo/q_t": 0.5010054707527161, + "grad_norm": 111.25892639160156, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.48374420404434204, + "logits/rejected": -0.4558669328689575, + "logps/chosen": -64.2926254272461, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.20619201660156, + "loss": 1.3915, + "margin_dpo/margin_mean": -0.029155343770980835, + "margin_dpo/margin_std": 0.516592800617218, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.03547096252441406, + "KL/mean": 0.0082835853099823, + "KL/rejected_KL_mean": 0.05204010009765625, + "KL/std": 0.2968295216560364, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.1428091824054718, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.08750605583190918, + "fcm_dpo/q_t": 0.5031182765960693, + "grad_norm": 122.44125366210938, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.49527525901794434, + "logits/rejected": -0.45649027824401855, + "logps/chosen": -58.14568328857422, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -103.99504089355469, + "loss": 1.3999, + "margin_dpo/margin_mean": -0.08750587701797485, + "margin_dpo/margin_std": 0.4646260738372803, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.014730453491210938, + "KL/mean": -0.025568902492523193, + "KL/rejected_KL_mean": -0.036407470703125, + "KL/std": 0.2190837562084198, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.14854301512241364, + "fcm_dpo/delta": 0.19682607054710388, + "fcm_dpo/margin": 0.021679282188415527, + "fcm_dpo/q_t": 0.49922505021095276, + "grad_norm": 95.31442260742188, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.4927077293395996, + "logits/rejected": -0.4740433096885681, + "logps/chosen": -56.98163986206055, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.84504699707031, + "loss": 1.3837, + "margin_dpo/margin_mean": 0.021679401397705078, + "margin_dpo/margin_std": 0.3349992036819458, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.0028209686279296875, + "KL/mean": -0.024948328733444214, + "KL/rejected_KL_mean": -0.047069549560546875, + "KL/std": 0.28548169136047363, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.15761470794677734, + "fcm_dpo/delta": 0.393257737159729, + "fcm_dpo/margin": 0.04425162076950073, + "fcm_dpo/q_t": 0.4983137249946594, + "grad_norm": 131.5380859375, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5590139031410217, + "logits/rejected": -0.5212494134902954, + "logps/chosen": -61.742713928222656, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.41654205322266, + "loss": 1.3802, + "margin_dpo/margin_mean": 0.04425153136253357, + "margin_dpo/margin_std": 0.38870832324028015, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.05492401123046875, + "KL/mean": -0.0016189813613891602, + "KL/rejected_KL_mean": -0.05816650390625, + "KL/std": 0.24518172442913055, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.1701841652393341, + "fcm_dpo/delta": 0.3815019130706787, + "fcm_dpo/margin": 0.1130858063697815, + "fcm_dpo/q_t": 0.49538034200668335, + "grad_norm": 132.9372100830078, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.4952155351638794, + "logits/rejected": -0.45743781328201294, + "logps/chosen": -67.65541076660156, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.43682098388672, + "loss": 1.3681, + "margin_dpo/margin_mean": 0.11308571696281433, + "margin_dpo/margin_std": 0.3566778898239136, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.014430999755859375, + "KL/mean": -0.03063647449016571, + "KL/rejected_KL_mean": -0.046840667724609375, + "KL/std": 0.24304544925689697, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.17673885822296143, + "fcm_dpo/delta": 0.18672488629817963, + "fcm_dpo/margin": 0.0324120819568634, + "fcm_dpo/q_t": 0.49859488010406494, + "grad_norm": 144.48635864257812, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.48481667041778564, + "logits/rejected": -0.4283279776573181, + "logps/chosen": -47.75392150878906, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.51912689208984, + "loss": 1.3812, + "margin_dpo/margin_mean": 0.03241202235221863, + "margin_dpo/margin_std": 0.341073215007782, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.011753082275390625, + "KL/mean": -0.04257369041442871, + "KL/rejected_KL_mean": -0.09689712524414062, + "KL/std": 0.22820980846881866, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.19068989157676697, + "fcm_dpo/delta": 0.3799416422843933, + "fcm_dpo/margin": 0.10864639282226562, + "fcm_dpo/q_t": 0.4949933886528015, + "grad_norm": 138.24989318847656, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.4894222915172577, + "logits/rejected": -0.439275860786438, + "logps/chosen": -70.193603515625, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.8544921875, + "loss": 1.3666, + "margin_dpo/margin_mean": 0.1086466908454895, + "margin_dpo/margin_std": 0.324674516916275, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.00156402587890625, + "KL/mean": -0.013564616441726685, + "KL/rejected_KL_mean": -0.0286865234375, + "KL/std": 0.26031258702278137, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.19809257984161377, + "fcm_dpo/delta": 0.19386807084083557, + "fcm_dpo/margin": 0.03026139736175537, + "fcm_dpo/q_t": 0.49853241443634033, + "grad_norm": 145.87615966796875, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5483527183532715, + "logits/rejected": -0.4915938377380371, + "logps/chosen": -50.80167770385742, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.85203552246094, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.030261576175689697, + "margin_dpo/margin_std": 0.3589157164096832, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.0018062591552734375, + "KL/mean": -0.06654942780733109, + "KL/rejected_KL_mean": -0.13129043579101562, + "KL/std": 0.30052614212036133, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.21362175047397614, + "fcm_dpo/delta": 0.3733234405517578, + "fcm_dpo/margin": 0.12948541343212128, + "fcm_dpo/q_t": 0.4933467209339142, + "grad_norm": 163.2720184326172, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.4900793731212616, + "logits/rejected": -0.4659076929092407, + "logps/chosen": -50.064823150634766, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -78.00008392333984, + "loss": 1.3605, + "margin_dpo/margin_mean": 0.1294853538274765, + "margin_dpo/margin_std": 0.3867538869380951, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.022380828857421875, + "KL/mean": -0.07612431049346924, + "KL/rejected_KL_mean": -0.17462539672851562, + "KL/std": 0.29673275351524353, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.22956205904483795, + "fcm_dpo/delta": 0.3563857674598694, + "fcm_dpo/margin": 0.1970118284225464, + "fcm_dpo/q_t": 0.48912686109542847, + "grad_norm": 189.14414978027344, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.5027599334716797, + "logits/rejected": -0.459952712059021, + "logps/chosen": -59.035255432128906, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.6792984008789, + "loss": 1.3441, + "margin_dpo/margin_mean": 0.19701123237609863, + "margin_dpo/margin_std": 0.4350666403770447, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.09295463562011719, + "KL/mean": -0.04670301079750061, + "KL/rejected_KL_mean": -0.18635940551757812, + "KL/std": 0.3279797434806824, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.2456461787223816, + "fcm_dpo/delta": 0.3337174952030182, + "fcm_dpo/margin": 0.2793177366256714, + "fcm_dpo/q_t": 0.4835093319416046, + "grad_norm": 189.1653594970703, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5013936161994934, + "logits/rejected": -0.4795346260070801, + "logps/chosen": -59.9847412109375, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.32591247558594, + "loss": 1.322, + "margin_dpo/margin_mean": 0.2793179750442505, + "margin_dpo/margin_std": 0.4338425397872925, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.05814552307128906, + "KL/mean": -0.08284728974103928, + "KL/rejected_KL_mean": -0.223846435546875, + "KL/std": 0.28930699825286865, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.26267242431640625, + "fcm_dpo/delta": 0.3280026316642761, + "fcm_dpo/margin": 0.2819862961769104, + "fcm_dpo/q_t": 0.4820657968521118, + "grad_norm": 217.61366271972656, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5185187458992004, + "logits/rejected": -0.5017420649528503, + "logps/chosen": -44.232887268066406, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.34906005859375, + "loss": 1.316, + "margin_dpo/margin_mean": 0.2819863259792328, + "margin_dpo/margin_std": 0.3861256241798401, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.02272796630859375, + "KL/mean": -0.11468243598937988, + "KL/rejected_KL_mean": -0.20663070678710938, + "KL/std": 0.34022825956344604, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.2813451886177063, + "fcm_dpo/delta": 0.34977710247039795, + "fcm_dpo/margin": 0.1839045286178589, + "fcm_dpo/q_t": 0.4874874949455261, + "grad_norm": 203.8140869140625, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.49157387018203735, + "logits/rejected": -0.461466521024704, + "logps/chosen": -52.55978012084961, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.54882049560547, + "loss": 1.3384, + "margin_dpo/margin_mean": 0.18390432000160217, + "margin_dpo/margin_std": 0.4098112881183624, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.1105499267578125, + "KL/mean": -0.10616210103034973, + "KL/rejected_KL_mean": -0.3228721618652344, + "KL/std": 0.4563215374946594, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.29814982414245605, + "fcm_dpo/delta": 0.2742401361465454, + "fcm_dpo/margin": 0.4334239363670349, + "fcm_dpo/q_t": 0.4688795804977417, + "grad_norm": 238.1178741455078, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5403963923454285, + "logits/rejected": -0.5089724063873291, + "logps/chosen": -53.812255859375, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.68258666992188, + "loss": 1.2685, + "margin_dpo/margin_mean": 0.43342408537864685, + "margin_dpo/margin_std": 0.5728694796562195, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.106048583984375, + "KL/mean": -0.14930874109268188, + "KL/rejected_KL_mean": -0.4046669006347656, + "KL/std": 0.47502797842025757, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.31298160552978516, + "fcm_dpo/delta": 0.24438421428203583, + "fcm_dpo/margin": 0.5107137560844421, + "fcm_dpo/q_t": 0.46163028478622437, + "grad_norm": 265.6593322753906, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5414735078811646, + "logits/rejected": -0.5054260492324829, + "logps/chosen": -42.79248046875, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.12886810302734, + "loss": 1.2417, + "margin_dpo/margin_mean": 0.5107135772705078, + "margin_dpo/margin_std": 0.5817660093307495, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.03438377380371094, + "KL/mean": -0.14049339294433594, + "KL/rejected_KL_mean": -0.24660491943359375, + "KL/std": 0.38090458512306213, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.33398109674453735, + "fcm_dpo/delta": 0.3309360146522522, + "fcm_dpo/margin": 0.21221554279327393, + "fcm_dpo/q_t": 0.48280662298202515, + "grad_norm": 246.491455078125, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.514916181564331, + "logits/rejected": -0.46059972047805786, + "logps/chosen": -60.590885162353516, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.64772033691406, + "loss": 1.3252, + "margin_dpo/margin_mean": 0.21221548318862915, + "margin_dpo/margin_std": 0.5604289770126343, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.14589309692382812, + "KL/mean": -0.13142457604408264, + "KL/rejected_KL_mean": -0.4087409973144531, + "KL/std": 0.4925822913646698, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.3499881625175476, + "fcm_dpo/delta": 0.20987650752067566, + "fcm_dpo/margin": 0.554635763168335, + "fcm_dpo/q_t": 0.4532009959220886, + "grad_norm": 283.0945129394531, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5688643455505371, + "logits/rejected": -0.5236915349960327, + "logps/chosen": -57.66189193725586, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.80309295654297, + "loss": 1.2119, + "margin_dpo/margin_mean": 0.5546364784240723, + "margin_dpo/margin_std": 0.5857715010643005, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.13712120056152344, + "KL/mean": -0.200497105717659, + "KL/rejected_KL_mean": -0.5381202697753906, + "KL/std": 0.5691412687301636, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.36175572872161865, + "fcm_dpo/delta": 0.15976664423942566, + "fcm_dpo/margin": 0.6752403974533081, + "fcm_dpo/q_t": 0.4409366846084595, + "grad_norm": 275.0484619140625, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.4905604124069214, + "logits/rejected": -0.4596910774707794, + "logps/chosen": -52.440250396728516, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.02732849121094, + "loss": 1.1678, + "margin_dpo/margin_mean": 0.6752407550811768, + "margin_dpo/margin_std": 0.5740634202957153, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.11575698852539062, + "KL/mean": -0.11299274861812592, + "KL/rejected_KL_mean": -0.34174346923828125, + "KL/std": 0.5324821472167969, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.37860026955604553, + "fcm_dpo/delta": 0.23025089502334595, + "fcm_dpo/margin": 0.4575031101703644, + "fcm_dpo/q_t": 0.4584744870662689, + "grad_norm": 232.2359619140625, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.4892122149467468, + "logits/rejected": -0.44255244731903076, + "logps/chosen": -63.691162109375, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.23574829101562, + "loss": 1.2351, + "margin_dpo/margin_mean": 0.4575028419494629, + "margin_dpo/margin_std": 0.6414620876312256, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.1771411895751953, + "KL/mean": -0.1902114897966385, + "KL/rejected_KL_mean": -0.5575637817382812, + "KL/std": 0.6925100088119507, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.38922953605651855, + "fcm_dpo/delta": 0.11731503903865814, + "fcm_dpo/margin": 0.7347012758255005, + "fcm_dpo/q_t": 0.43250370025634766, + "grad_norm": 264.69793701171875, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5206853151321411, + "logits/rejected": -0.4798637628555298, + "logps/chosen": -62.56238555908203, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.87506103515625, + "loss": 1.1484, + "margin_dpo/margin_mean": 0.7347011566162109, + "margin_dpo/margin_std": 0.88288414478302, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.09492874145507812, + "KL/mean": -0.19503909349441528, + "KL/rejected_KL_mean": -0.4850044250488281, + "KL/std": 0.522113561630249, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.4027722477912903, + "fcm_dpo/delta": 0.16955889761447906, + "fcm_dpo/margin": 0.5799333453178406, + "fcm_dpo/q_t": 0.443417489528656, + "grad_norm": 255.80274963378906, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.47367024421691895, + "logits/rejected": -0.4471021294593811, + "logps/chosen": -53.166046142578125, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.3701400756836, + "loss": 1.1797, + "margin_dpo/margin_mean": 0.5799335241317749, + "margin_dpo/margin_std": 0.5616409182548523, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.08447456359863281, + "KL/mean": -0.3024243116378784, + "KL/rejected_KL_mean": -0.6893310546875, + "KL/std": 0.7118836641311646, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.40975314378738403, + "fcm_dpo/delta": 0.08569816499948502, + "fcm_dpo/margin": 0.7737990021705627, + "fcm_dpo/q_t": 0.4249619245529175, + "grad_norm": 254.81643676757812, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.497056245803833, + "logits/rejected": -0.4796808958053589, + "logps/chosen": -50.732852935791016, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.61117553710938, + "loss": 1.1261, + "margin_dpo/margin_mean": 0.7737988233566284, + "margin_dpo/margin_std": 0.8890845775604248, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.13554954528808594, + "KL/mean": -0.4650111794471741, + "KL/rejected_KL_mean": -1.0655708312988281, + "KL/std": 0.9833367466926575, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.405403733253479, + "fcm_dpo/delta": -0.09166581928730011, + "fcm_dpo/margin": 1.2011213302612305, + "fcm_dpo/q_t": 0.38606151938438416, + "grad_norm": 238.96286010742188, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5317063927650452, + "logits/rejected": -0.49485355615615845, + "logps/chosen": -50.888938903808594, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.88999938964844, + "loss": 0.9969, + "margin_dpo/margin_mean": 1.2011216878890991, + "margin_dpo/margin_std": 1.0503125190734863, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.022670745849609375, + "KL/mean": -0.5429773330688477, + "KL/rejected_KL_mean": -1.1086235046386719, + "KL/std": 1.059760332107544, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.40310993790626526, + "fcm_dpo/delta": -0.05904524773359299, + "fcm_dpo/margin": 1.1312965154647827, + "fcm_dpo/q_t": 0.3944385349750519, + "grad_norm": 202.62318420410156, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5578644275665283, + "logits/rejected": -0.5216199159622192, + "logps/chosen": -51.96881866455078, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.14924621582031, + "loss": 1.0416, + "margin_dpo/margin_mean": 1.1312960386276245, + "margin_dpo/margin_std": 1.2232028245925903, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.008932113647460938, + "KL/mean": -0.49562323093414307, + "KL/rejected_KL_mean": -1.0001792907714844, + "KL/std": 1.0603375434875488, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.39714089035987854, + "fcm_dpo/delta": -0.0015208348631858826, + "fcm_dpo/margin": 1.0091136693954468, + "fcm_dpo/q_t": 0.40975332260131836, + "grad_norm": 183.8545379638672, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5097329616546631, + "logits/rejected": -0.466074675321579, + "logps/chosen": -62.79817581176758, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.89524841308594, + "loss": 1.0959, + "margin_dpo/margin_mean": 1.00911283493042, + "margin_dpo/margin_std": 1.3811231851577759, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1762561798095703, + "KL/mean": -0.5279443264007568, + "KL/rejected_KL_mean": -1.232147216796875, + "KL/std": 1.3923825025558472, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.39080196619033813, + "fcm_dpo/delta": -0.15896156430244446, + "fcm_dpo/margin": 1.4084022045135498, + "fcm_dpo/q_t": 0.3806955814361572, + "grad_norm": 187.6940155029297, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5100470781326294, + "logits/rejected": -0.477189302444458, + "logps/chosen": -48.214263916015625, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.14459228515625, + "loss": 1.0018, + "margin_dpo/margin_mean": 1.408402681350708, + "margin_dpo/margin_std": 1.7267735004425049, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.08431053161621094, + "KL/mean": -0.726874053478241, + "KL/rejected_KL_mean": -1.538055419921875, + "KL/std": 1.270685076713562, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.3733319044113159, + "fcm_dpo/delta": -0.21987757086753845, + "fcm_dpo/margin": 1.6223597526550293, + "fcm_dpo/q_t": 0.3587682843208313, + "grad_norm": 187.23959350585938, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5531972646713257, + "logits/rejected": -0.512998104095459, + "logps/chosen": -50.666160583496094, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.1075668334961, + "loss": 0.9274, + "margin_dpo/margin_mean": 1.6223597526550293, + "margin_dpo/margin_std": 1.3455851078033447, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.20937728881835938, + "KL/mean": -0.5704580545425415, + "KL/rejected_KL_mean": -1.3502960205078125, + "KL/std": 1.3929988145828247, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.36257779598236084, + "fcm_dpo/delta": -0.17505738139152527, + "fcm_dpo/margin": 1.5596754550933838, + "fcm_dpo/q_t": 0.3747410774230957, + "grad_norm": 151.57008361816406, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5110136270523071, + "logits/rejected": -0.48049166798591614, + "logps/chosen": -57.77568817138672, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.6503677368164, + "loss": 0.981, + "margin_dpo/margin_mean": 1.559675931930542, + "margin_dpo/margin_std": 1.6767141819000244, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.024274826049804688, + "KL/mean": -0.8587805032730103, + "KL/rejected_KL_mean": -1.7418327331542969, + "KL/std": 1.7877637147903442, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.3462945222854614, + "fcm_dpo/delta": -0.22512511909008026, + "fcm_dpo/margin": 1.766108512878418, + "fcm_dpo/q_t": 0.3654175400733948, + "grad_norm": 164.017333984375, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5302759408950806, + "logits/rejected": -0.4932605028152466, + "logps/chosen": -62.671539306640625, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.76536560058594, + "loss": 0.966, + "margin_dpo/margin_mean": 1.7661091089248657, + "margin_dpo/margin_std": 1.9768691062927246, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.24113082885742188, + "KL/mean": -1.0401140451431274, + "KL/rejected_KL_mean": -2.3213577270507812, + "KL/std": 2.1446995735168457, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.3195430636405945, + "fcm_dpo/delta": -0.4562838673591614, + "fcm_dpo/margin": 2.562486171722412, + "fcm_dpo/q_t": 0.3244972229003906, + "grad_norm": 149.43026733398438, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5593968033790588, + "logits/rejected": -0.513144314289093, + "logps/chosen": -58.72529602050781, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.229736328125, + "loss": 0.8416, + "margin_dpo/margin_mean": 2.562486410140991, + "margin_dpo/margin_std": 2.4132444858551025, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5456638336181641, + "KL/mean": -0.698434591293335, + "KL/rejected_KL_mean": -1.942535400390625, + "KL/std": 1.798791766166687, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.29444074630737305, + "fcm_dpo/delta": -0.361034095287323, + "fcm_dpo/margin": 2.4881997108459473, + "fcm_dpo/q_t": 0.33181002736091614, + "grad_norm": 141.6117401123047, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5517200827598572, + "logits/rejected": -0.5274295806884766, + "logps/chosen": -53.61033248901367, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.4227294921875, + "loss": 0.8477, + "margin_dpo/margin_mean": 2.4881997108459473, + "margin_dpo/margin_std": 1.9044840335845947, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.24141502380371094, + "KL/mean": -1.1338403224945068, + "KL/rejected_KL_mean": -2.50909423828125, + "KL/std": 2.213291645050049, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.27339714765548706, + "fcm_dpo/delta": -0.3821863532066345, + "fcm_dpo/margin": 2.7505087852478027, + "fcm_dpo/q_t": 0.3291955888271332, + "grad_norm": 141.95986938476562, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.48910439014434814, + "logits/rejected": -0.46865081787109375, + "logps/chosen": -49.83708190917969, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.29285430908203, + "loss": 0.8437, + "margin_dpo/margin_mean": 2.7505087852478027, + "margin_dpo/margin_std": 2.1812946796417236, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.13675880432128906, + "KL/mean": -0.9691852927207947, + "KL/rejected_KL_mean": -2.0751304626464844, + "KL/std": 2.018606185913086, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.26203739643096924, + "fcm_dpo/delta": -0.19032379984855652, + "fcm_dpo/margin": 2.2118911743164062, + "fcm_dpo/q_t": 0.37283578515052795, + "grad_norm": 114.09217834472656, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4807615280151367, + "logits/rejected": -0.46815767884254456, + "logps/chosen": -48.278167724609375, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.01155853271484, + "loss": 0.9793, + "margin_dpo/margin_mean": 2.2118911743164062, + "margin_dpo/margin_std": 2.5098156929016113, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.1975116729736328, + "KL/mean": -1.2001652717590332, + "KL/rejected_KL_mean": -2.5978469848632812, + "KL/std": 2.51208233833313, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.24773001670837402, + "fcm_dpo/delta": -0.3136446177959442, + "fcm_dpo/margin": 2.7953567504882812, + "fcm_dpo/q_t": 0.35114121437072754, + "grad_norm": 121.18917083740234, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5210041999816895, + "logits/rejected": -0.4696595072746277, + "logps/chosen": -55.80191421508789, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.25043487548828, + "loss": 0.9241, + "margin_dpo/margin_mean": 2.795356512069702, + "margin_dpo/margin_std": 3.0661070346832275, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.3682823181152344, + "KL/mean": -1.053499460220337, + "KL/rejected_KL_mean": -2.47528076171875, + "KL/std": 2.465869903564453, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.23447749018669128, + "fcm_dpo/delta": -0.2848522365093231, + "fcm_dpo/margin": 2.843561887741089, + "fcm_dpo/q_t": 0.3497909605503082, + "grad_norm": 113.97206115722656, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5685409307479858, + "logits/rejected": -0.5155435800552368, + "logps/chosen": -57.557796478271484, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.15448760986328, + "loss": 0.9085, + "margin_dpo/margin_mean": 2.8435616493225098, + "margin_dpo/margin_std": 2.557325601577759, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.10150527954101562, + "KL/mean": -1.302559494972229, + "KL/rejected_KL_mean": -2.706623077392578, + "KL/std": 2.41989803314209, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.2204352766275406, + "fcm_dpo/delta": -0.23567046225070953, + "fcm_dpo/margin": 2.808133840560913, + "fcm_dpo/q_t": 0.35952016711235046, + "grad_norm": 121.0833969116211, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5958759784698486, + "logits/rejected": -0.5383877754211426, + "logps/chosen": -57.086570739746094, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -90.72322082519531, + "loss": 0.946, + "margin_dpo/margin_mean": 2.808133840560913, + "margin_dpo/margin_std": 2.610079288482666, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.37255859375, + "KL/mean": -1.3447418212890625, + "KL/rejected_KL_mean": -3.0620384216308594, + "KL/std": 3.0294926166534424, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.2095491588115692, + "fcm_dpo/delta": -0.34319785237312317, + "fcm_dpo/margin": 3.4345943927764893, + "fcm_dpo/q_t": 0.3436010777950287, + "grad_norm": 239.5515594482422, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5714051723480225, + "logits/rejected": -0.5157973766326904, + "logps/chosen": -61.312713623046875, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -86.82951354980469, + "loss": 0.8994, + "margin_dpo/margin_mean": 3.4345943927764893, + "margin_dpo/margin_std": 3.425793170928955, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.06844711303710938, + "KL/mean": -1.9518392086029053, + "KL/rejected_KL_mean": -3.8352279663085938, + "KL/std": 3.1761012077331543, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.1934700906276703, + "fcm_dpo/delta": -0.3559325337409973, + "fcm_dpo/margin": 3.766773223876953, + "fcm_dpo/q_t": 0.339402437210083, + "grad_norm": 92.9075927734375, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.539138913154602, + "logits/rejected": -0.5032519102096558, + "logps/chosen": -58.79258728027344, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.19337463378906, + "loss": 0.8891, + "margin_dpo/margin_mean": 3.766773223876953, + "margin_dpo/margin_std": 3.6567764282226562, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.17328453063964844, + "KL/mean": -2.029240608215332, + "KL/rejected_KL_mean": -3.8852005004882812, + "KL/std": 3.772291898727417, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.18080684542655945, + "fcm_dpo/delta": -0.29471051692962646, + "fcm_dpo/margin": 3.7119140625, + "fcm_dpo/q_t": 0.3605567514896393, + "grad_norm": 75.60486602783203, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5066142082214355, + "logits/rejected": -0.47301632165908813, + "logps/chosen": -61.54695129394531, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -79.88720703125, + "loss": 0.9632, + "margin_dpo/margin_mean": 3.711913585662842, + "margin_dpo/margin_std": 4.616769790649414, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.46772193908691406, + "KL/mean": -2.334967851638794, + "KL/rejected_KL_mean": -5.137657165527344, + "KL/std": 4.306759834289551, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.16471263766288757, + "fcm_dpo/delta": -0.5775514841079712, + "fcm_dpo/margin": 5.60538387298584, + "fcm_dpo/q_t": 0.2996635138988495, + "grad_norm": 75.43959045410156, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5387547016143799, + "logits/rejected": -0.4824455976486206, + "logps/chosen": -51.86963653564453, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.111572265625, + "loss": 0.7747, + "margin_dpo/margin_mean": 5.60538387298584, + "margin_dpo/margin_std": 4.600427627563477, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.06701278686523438, + "KL/mean": -2.89475154876709, + "KL/rejected_KL_mean": -5.7224884033203125, + "KL/std": 4.909053325653076, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.15035340189933777, + "fcm_dpo/delta": -0.48940175771713257, + "fcm_dpo/margin": 5.655482769012451, + "fcm_dpo/q_t": 0.3268622159957886, + "grad_norm": 74.27436828613281, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6001520752906799, + "logits/rejected": -0.5782574415206909, + "logps/chosen": -53.38166427612305, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -97.50608825683594, + "loss": 0.8546, + "margin_dpo/margin_mean": 5.655483245849609, + "margin_dpo/margin_std": 5.626848220825195, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.19623184204101562, + "KL/mean": -2.7065939903259277, + "KL/rejected_KL_mean": -5.216953277587891, + "KL/std": 4.595136642456055, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.13865108788013458, + "fcm_dpo/delta": -0.31784510612487793, + "fcm_dpo/margin": 5.020720958709717, + "fcm_dpo/q_t": 0.34497055411338806, + "grad_norm": 65.8781509399414, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.604508638381958, + "logits/rejected": -0.5513991117477417, + "logps/chosen": -50.88489532470703, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -96.93234252929688, + "loss": 0.8905, + "margin_dpo/margin_mean": 5.020720481872559, + "margin_dpo/margin_std": 4.618564605712891, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.6595840454101562, + "KL/mean": -3.6356935501098633, + "KL/rejected_KL_mean": -6.611797332763672, + "KL/std": 5.609594345092773, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.12890280783176422, + "fcm_dpo/delta": -0.3965994417667389, + "fcm_dpo/margin": 5.952215194702148, + "fcm_dpo/q_t": 0.3394591212272644, + "grad_norm": 64.30790710449219, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6537004709243774, + "logits/rejected": -0.5922633409500122, + "logps/chosen": -63.274818420410156, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -95.60530090332031, + "loss": 0.91, + "margin_dpo/margin_mean": 5.952215671539307, + "margin_dpo/margin_std": 6.726306915283203, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.4476184844970703, + "KL/mean": -3.2253758907318115, + "KL/rejected_KL_mean": -6.003135681152344, + "KL/std": 5.326668739318848, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.12058769166469574, + "fcm_dpo/delta": -0.28913071751594543, + "fcm_dpo/margin": 5.5555219650268555, + "fcm_dpo/q_t": 0.3563534617424011, + "grad_norm": 53.888919830322266, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.611944854259491, + "logits/rejected": -0.5693808794021606, + "logps/chosen": -58.380348205566406, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -100.17757415771484, + "loss": 0.9544, + "margin_dpo/margin_mean": 5.555521011352539, + "margin_dpo/margin_std": 6.480095386505127, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.4867591857910156, + "KL/mean": -3.617406129837036, + "KL/rejected_KL_mean": -6.748050689697266, + "KL/std": 5.308437824249268, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.11351069808006287, + "fcm_dpo/delta": -0.3334280550479889, + "fcm_dpo/margin": 6.261290550231934, + "fcm_dpo/q_t": 0.3403124213218689, + "grad_norm": 59.81298828125, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5721638798713684, + "logits/rejected": -0.5439426898956299, + "logps/chosen": -70.98204040527344, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -102.31350708007812, + "loss": 0.8916, + "margin_dpo/margin_mean": 6.261290550231934, + "margin_dpo/margin_std": 5.751809120178223, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.6511821746826172, + "KL/mean": -4.1788763999938965, + "KL/rejected_KL_mean": -7.706577301025391, + "KL/std": 6.134248733520508, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.105606809258461, + "fcm_dpo/delta": -0.37162643671035767, + "fcm_dpo/margin": 7.055391311645508, + "fcm_dpo/q_t": 0.34046316146850586, + "grad_norm": 60.099647521972656, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.583040714263916, + "logits/rejected": -0.5060716271400452, + "logps/chosen": -62.784122467041016, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -92.32386779785156, + "loss": 0.9001, + "margin_dpo/margin_mean": 7.055391311645508, + "margin_dpo/margin_std": 7.35453987121582, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.0634136199951172, + "KL/mean": -4.876946449279785, + "KL/rejected_KL_mean": -8.690475463867188, + "KL/std": 6.619156837463379, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.09668101370334625, + "fcm_dpo/delta": -0.36965954303741455, + "fcm_dpo/margin": 7.6270647048950195, + "fcm_dpo/q_t": 0.34003186225891113, + "grad_norm": 55.22077560424805, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6169421672821045, + "logits/rejected": -0.5754032135009766, + "logps/chosen": -52.995941162109375, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -97.57567596435547, + "loss": 0.8956, + "margin_dpo/margin_mean": 7.6270647048950195, + "margin_dpo/margin_std": 7.726261138916016, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.0238513946533203, + "KL/mean": -5.118398666381836, + "KL/rejected_KL_mean": -8.212940216064453, + "KL/std": 5.96435546875, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.09275862574577332, + "fcm_dpo/delta": -0.1858130246400833, + "fcm_dpo/margin": 6.189088821411133, + "fcm_dpo/q_t": 0.36864781379699707, + "grad_norm": 60.84640121459961, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.614082396030426, + "logits/rejected": -0.5546412467956543, + "logps/chosen": -62.966041564941406, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -93.60633850097656, + "loss": 0.9832, + "margin_dpo/margin_mean": 6.189088821411133, + "margin_dpo/margin_std": 6.81551456451416, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.0461502075195312, + "KL/mean": -4.7814412117004395, + "KL/rejected_KL_mean": -8.516727447509766, + "KL/std": 8.099505424499512, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.08813193440437317, + "fcm_dpo/delta": -0.2782770097255707, + "fcm_dpo/margin": 7.470581531524658, + "fcm_dpo/q_t": 0.3644851744174957, + "grad_norm": 49.49559020996094, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.5756335854530334, + "logits/rejected": -0.540107011795044, + "logps/chosen": -61.67967224121094, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -98.36922454833984, + "loss": 0.9752, + "margin_dpo/margin_mean": 7.470581531524658, + "margin_dpo/margin_std": 9.972640991210938, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.217864990234375, + "KL/mean": -4.094658374786377, + "KL/rejected_KL_mean": -6.9714508056640625, + "KL/std": 5.886908054351807, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.08628196269273758, + "fcm_dpo/delta": -0.10137659311294556, + "fcm_dpo/margin": 5.7535834312438965, + "fcm_dpo/q_t": 0.3890373706817627, + "grad_norm": 47.91719055175781, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.5558626055717468, + "logits/rejected": -0.5187328457832336, + "logps/chosen": -57.36863708496094, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -82.53764343261719, + "loss": 1.0359, + "margin_dpo/margin_mean": 5.753582954406738, + "margin_dpo/margin_std": 7.107942581176758, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.094423294067383, + "KL/mean": -5.968453407287598, + "KL/rejected_KL_mean": -9.842483520507812, + "KL/std": 7.609671592712402, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.08184128254652023, + "fcm_dpo/delta": -0.25242727994918823, + "fcm_dpo/margin": 7.748061180114746, + "fcm_dpo/q_t": 0.3566237688064575, + "grad_norm": 51.09866714477539, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6205369830131531, + "logits/rejected": -0.5761100053787231, + "logps/chosen": -75.2418212890625, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -107.45254516601562, + "loss": 0.945, + "margin_dpo/margin_mean": 7.748061656951904, + "margin_dpo/margin_std": 7.969051361083984, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.6668148040771484, + "KL/mean": -5.147095680236816, + "KL/rejected_KL_mean": -9.62738037109375, + "KL/std": 8.096553802490234, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.07678164541721344, + "fcm_dpo/delta": -0.31488358974456787, + "fcm_dpo/margin": 8.960559844970703, + "fcm_dpo/q_t": 0.3502163887023926, + "grad_norm": 46.31606674194336, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5900973677635193, + "logits/rejected": -0.5584604740142822, + "logps/chosen": -54.66541290283203, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -103.15757751464844, + "loss": 0.9323, + "margin_dpo/margin_mean": 8.960559844970703, + "margin_dpo/margin_std": 9.844915390014648, + "step": 64 + }, + { + "KL/chosen_KL_mean": -2.6340904235839844, + "KL/mean": -7.138725280761719, + "KL/rejected_KL_mean": -11.643363952636719, + "KL/std": 9.097021102905273, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.07288145273923874, + "fcm_dpo/delta": -0.2767026424407959, + "fcm_dpo/margin": 9.009276390075684, + "fcm_dpo/q_t": 0.3535335063934326, + "grad_norm": 45.35810852050781, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6539331674575806, + "logits/rejected": -0.6399848461151123, + "logps/chosen": -67.4700927734375, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -121.58982849121094, + "loss": 0.948, + "margin_dpo/margin_mean": 9.009276390075684, + "margin_dpo/margin_std": 9.86502456665039, + "step": 65 + }, + { + "KL/chosen_KL_mean": -2.416036605834961, + "KL/mean": -6.624704360961914, + "KL/rejected_KL_mean": -10.833366394042969, + "KL/std": 8.27562141418457, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.06959041953086853, + "fcm_dpo/delta": -0.19998188316822052, + "fcm_dpo/margin": 8.417326927185059, + "fcm_dpo/q_t": 0.3716912567615509, + "grad_norm": 41.684532165527344, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6341314315795898, + "logits/rejected": -0.6006229519844055, + "logps/chosen": -53.859561920166016, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -86.46966552734375, + "loss": 0.9845, + "margin_dpo/margin_mean": 8.417326927185059, + "margin_dpo/margin_std": 9.898199081420898, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.1657180786132812, + "KL/mean": -6.501282691955566, + "KL/rejected_KL_mean": -10.83685302734375, + "KL/std": 8.343762397766113, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.06757616996765137, + "fcm_dpo/delta": -0.19716030359268188, + "fcm_dpo/margin": 8.671130180358887, + "fcm_dpo/q_t": 0.3712519109249115, + "grad_norm": 39.73466491699219, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6380045413970947, + "logits/rejected": -0.5994296669960022, + "logps/chosen": -61.50652313232422, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -83.62413787841797, + "loss": 0.9815, + "margin_dpo/margin_mean": 8.67112922668457, + "margin_dpo/margin_std": 10.073928833007812, + "step": 67 + }, + { + "KL/chosen_KL_mean": -2.3892745971679688, + "KL/mean": -6.534235954284668, + "KL/rejected_KL_mean": -10.67919921875, + "KL/std": 7.522188663482666, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.06540803611278534, + "fcm_dpo/delta": -0.15013387799263, + "fcm_dpo/margin": 8.289920806884766, + "fcm_dpo/q_t": 0.37633758783340454, + "grad_norm": 39.29888153076172, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6329935193061829, + "logits/rejected": -0.5741623640060425, + "logps/chosen": -67.59510803222656, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -87.88644409179688, + "loss": 0.9835, + "margin_dpo/margin_mean": 8.289920806884766, + "margin_dpo/margin_std": 8.395111083984375, + "step": 68 + }, + { + "KL/chosen_KL_mean": -2.8640060424804688, + "KL/mean": -7.839422702789307, + "KL/rejected_KL_mean": -12.81484603881836, + "KL/std": 8.856003761291504, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.06224876642227173, + "fcm_dpo/delta": -0.23532283306121826, + "fcm_dpo/margin": 9.950838088989258, + "fcm_dpo/q_t": 0.360681414604187, + "grad_norm": 41.4715576171875, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6257553100585938, + "logits/rejected": -0.6013358235359192, + "logps/chosen": -62.68324661254883, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -116.20370483398438, + "loss": 0.9375, + "margin_dpo/margin_mean": 9.950839042663574, + "margin_dpo/margin_std": 9.716062545776367, + "step": 69 + }, + { + "KL/chosen_KL_mean": -4.179302215576172, + "KL/mean": -9.5701904296875, + "KL/rejected_KL_mean": -14.961078643798828, + "KL/std": 10.651983261108398, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.05961456894874573, + "fcm_dpo/delta": -0.2588519752025604, + "fcm_dpo/margin": 10.781774520874023, + "fcm_dpo/q_t": 0.35821232199668884, + "grad_norm": 42.19196319580078, + "learning_rate": 5e-07, + "logits/chosen": -0.5981051921844482, + "logits/rejected": -0.5617469549179077, + "logps/chosen": -66.10993957519531, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -106.02186584472656, + "loss": 0.9421, + "margin_dpo/margin_mean": 10.781774520874023, + "margin_dpo/margin_std": 11.324002265930176, + "step": 70 + }, + { + "KL/chosen_KL_mean": -3.783407211303711, + "KL/mean": -10.018430709838867, + "KL/rejected_KL_mean": -16.253456115722656, + "KL/std": 10.964170455932617, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.05581720173358917, + "fcm_dpo/delta": -0.3190196752548218, + "fcm_dpo/margin": 12.470056533813477, + "fcm_dpo/q_t": 0.3471716046333313, + "grad_norm": 38.125030517578125, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6462788581848145, + "logits/rejected": -0.6049303412437439, + "logps/chosen": -65.53374481201172, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -113.59007263183594, + "loss": 0.9019, + "margin_dpo/margin_mean": 12.47005558013916, + "margin_dpo/margin_std": 11.962203979492188, + "step": 71 + }, + { + "KL/chosen_KL_mean": -4.315851211547852, + "KL/mean": -10.979276657104492, + "KL/rejected_KL_mean": -17.6427001953125, + "KL/std": 11.922504425048828, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.05232307314872742, + "fcm_dpo/delta": -0.3206704258918762, + "fcm_dpo/margin": 13.326845169067383, + "fcm_dpo/q_t": 0.34632056951522827, + "grad_norm": 37.637332916259766, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6497834920883179, + "logits/rejected": -0.6138608455657959, + "logps/chosen": -70.3692626953125, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -112.9296875, + "loss": 0.9126, + "margin_dpo/margin_mean": 13.326844215393066, + "margin_dpo/margin_std": 13.707250595092773, + "step": 72 + }, + { + "KL/chosen_KL_mean": -6.226812362670898, + "KL/mean": -12.874276161193848, + "KL/rejected_KL_mean": -19.521743774414062, + "KL/std": 15.596323013305664, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.04931444674730301, + "fcm_dpo/delta": -0.27526822686195374, + "fcm_dpo/margin": 13.294928550720215, + "fcm_dpo/q_t": 0.36772334575653076, + "grad_norm": 37.14633560180664, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6717506051063538, + "logits/rejected": -0.6292352676391602, + "logps/chosen": -72.48308563232422, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -109.97787475585938, + "loss": 1.0164, + "margin_dpo/margin_mean": 13.294928550720215, + "margin_dpo/margin_std": 19.35903549194336, + "step": 73 + }, + { + "KL/chosen_KL_mean": -6.872499465942383, + "KL/mean": -14.145965576171875, + "KL/rejected_KL_mean": -21.41943359375, + "KL/std": 17.387737274169922, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.04657856374979019, + "fcm_dpo/delta": -0.29889941215515137, + "fcm_dpo/margin": 14.546926498413086, + "fcm_dpo/q_t": 0.36223089694976807, + "grad_norm": 37.54848861694336, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6387256979942322, + "logits/rejected": -0.6279802322387695, + "logps/chosen": -60.29738235473633, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -117.36637115478516, + "loss": 0.9876, + "margin_dpo/margin_mean": 14.546926498413086, + "margin_dpo/margin_std": 21.452259063720703, + "step": 74 + }, + { + "KL/chosen_KL_mean": -6.849504470825195, + "KL/mean": -17.168128967285156, + "KL/rejected_KL_mean": -27.486740112304688, + "KL/std": 18.24625015258789, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.042833440005779266, + "fcm_dpo/delta": -0.5284513235092163, + "fcm_dpo/margin": 20.637245178222656, + "fcm_dpo/q_t": 0.31754931807518005, + "grad_norm": 33.54880905151367, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6500366926193237, + "logits/rejected": -0.6410285830497742, + "logps/chosen": -58.71116638183594, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -138.74072265625, + "loss": 0.8367, + "margin_dpo/margin_mean": 20.637245178222656, + "margin_dpo/margin_std": 20.518566131591797, + "step": 75 + }, + { + "KL/chosen_KL_mean": -8.56123161315918, + "KL/mean": -15.246042251586914, + "KL/rejected_KL_mean": -21.93084716796875, + "KL/std": 14.937604904174805, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.04049314558506012, + "fcm_dpo/delta": -0.14943483471870422, + "fcm_dpo/margin": 13.369623184204102, + "fcm_dpo/q_t": 0.3767305314540863, + "grad_norm": 33.598426818847656, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.6957746744155884, + "logits/rejected": -0.6527870893478394, + "logps/chosen": -61.8272705078125, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -100.14747619628906, + "loss": 1.0108, + "margin_dpo/margin_mean": 13.369623184204102, + "margin_dpo/margin_std": 16.221744537353516, + "step": 76 + }, + { + "KL/chosen_KL_mean": -7.690212249755859, + "KL/mean": -18.52901840209961, + "KL/rejected_KL_mean": -29.36782455444336, + "KL/std": 20.59722137451172, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.03732946142554283, + "fcm_dpo/delta": -0.4494476318359375, + "fcm_dpo/margin": 21.677608489990234, + "fcm_dpo/q_t": 0.3250860571861267, + "grad_norm": 34.511131286621094, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6748108863830566, + "logits/rejected": -0.6345440745353699, + "logps/chosen": -65.78689575195312, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -123.14143371582031, + "loss": 0.8778, + "margin_dpo/margin_mean": 21.677608489990234, + "margin_dpo/margin_std": 22.455432891845703, + "step": 77 + }, + { + "KL/chosen_KL_mean": -7.874031066894531, + "KL/mean": -16.227333068847656, + "KL/rejected_KL_mean": -24.58063507080078, + "KL/std": 16.826889038085938, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.03571484982967377, + "fcm_dpo/delta": -0.2086990475654602, + "fcm_dpo/margin": 16.70660400390625, + "fcm_dpo/q_t": 0.36868858337402344, + "grad_norm": 31.562875747680664, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6353791952133179, + "logits/rejected": -0.6087647676467896, + "logps/chosen": -63.487815856933594, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -109.51499938964844, + "loss": 0.9795, + "margin_dpo/margin_mean": 16.70660400390625, + "margin_dpo/margin_std": 19.152223587036133, + "step": 78 + }, + { + "KL/chosen_KL_mean": -8.209478378295898, + "KL/mean": -17.424894332885742, + "KL/rejected_KL_mean": -26.640308380126953, + "KL/std": 19.277175903320312, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.033957213163375854, + "fcm_dpo/delta": -0.24112707376480103, + "fcm_dpo/margin": 18.430828094482422, + "fcm_dpo/q_t": 0.3686911463737488, + "grad_norm": 27.449581146240234, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6632182598114014, + "logits/rejected": -0.6331592798233032, + "logps/chosen": -63.65996551513672, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -114.28787231445312, + "loss": 0.9901, + "margin_dpo/margin_mean": 18.430828094482422, + "margin_dpo/margin_std": 23.57292938232422, + "step": 79 + }, + { + "KL/chosen_KL_mean": -11.220630645751953, + "KL/mean": -19.71291732788086, + "KL/rejected_KL_mean": -28.20520782470703, + "KL/std": 20.231279373168945, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.03248097002506256, + "fcm_dpo/delta": -0.16335377097129822, + "fcm_dpo/margin": 16.98456573486328, + "fcm_dpo/q_t": 0.38458332419395447, + "grad_norm": 30.238367080688477, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.722549557685852, + "logits/rejected": -0.6791394948959351, + "logps/chosen": -69.73992156982422, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -115.7527084350586, + "loss": 1.038, + "margin_dpo/margin_mean": 16.984567642211914, + "margin_dpo/margin_std": 23.31180763244629, + "step": 80 + }, + { + "KL/chosen_KL_mean": -10.386722564697266, + "KL/mean": -22.262874603271484, + "KL/rejected_KL_mean": -34.13903045654297, + "KL/std": 24.364049911499023, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.03061431273818016, + "fcm_dpo/delta": -0.3560195863246918, + "fcm_dpo/margin": 23.752304077148438, + "fcm_dpo/q_t": 0.3488122224807739, + "grad_norm": 30.49542808532715, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7048654556274414, + "logits/rejected": -0.6742877960205078, + "logps/chosen": -76.83558654785156, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -163.80172729492188, + "loss": 0.9377, + "margin_dpo/margin_mean": 23.75230598449707, + "margin_dpo/margin_std": 28.99235725402832, + "step": 81 + }, + { + "KL/chosen_KL_mean": -12.659355163574219, + "KL/mean": -22.106792449951172, + "KL/rejected_KL_mean": -31.554229736328125, + "KL/std": 22.128211975097656, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.029791174456477165, + "fcm_dpo/delta": -0.17301397025585175, + "fcm_dpo/margin": 18.89487648010254, + "fcm_dpo/q_t": 0.38285931944847107, + "grad_norm": 32.891151428222656, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.646277666091919, + "logits/rejected": -0.6339075565338135, + "logps/chosen": -64.89173889160156, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -122.2974853515625, + "loss": 1.0766, + "margin_dpo/margin_mean": 18.894874572753906, + "margin_dpo/margin_std": 30.41067123413086, + "step": 82 + }, + { + "KL/chosen_KL_mean": -13.092794418334961, + "KL/mean": -24.271024703979492, + "KL/rejected_KL_mean": -35.44925308227539, + "KL/std": 23.241058349609375, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.028085583820939064, + "fcm_dpo/delta": -0.24422822892665863, + "fcm_dpo/margin": 22.356460571289062, + "fcm_dpo/q_t": 0.36477890610694885, + "grad_norm": 29.858074188232422, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6767470836639404, + "logits/rejected": -0.6405803561210632, + "logps/chosen": -68.92018127441406, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -139.1651611328125, + "loss": 0.981, + "margin_dpo/margin_mean": 22.356460571289062, + "margin_dpo/margin_std": 27.430038452148438, + "step": 83 + }, + { + "KL/chosen_KL_mean": -12.076467514038086, + "KL/mean": -21.793697357177734, + "KL/rejected_KL_mean": -31.510929107666016, + "KL/std": 19.938434600830078, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.027056865394115448, + "fcm_dpo/delta": -0.13519904017448425, + "fcm_dpo/margin": 19.43446159362793, + "fcm_dpo/q_t": 0.3784554600715637, + "grad_norm": 26.61237335205078, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6591858267784119, + "logits/rejected": -0.6200757026672363, + "logps/chosen": -79.25263977050781, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -118.80953216552734, + "loss": 1.0014, + "margin_dpo/margin_mean": 19.434459686279297, + "margin_dpo/margin_std": 20.418224334716797, + "step": 84 + }, + { + "KL/chosen_KL_mean": -12.675928115844727, + "KL/mean": -22.654075622558594, + "KL/rejected_KL_mean": -32.63222122192383, + "KL/std": 20.563871383666992, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.026730665937066078, + "fcm_dpo/delta": -0.1410984843969345, + "fcm_dpo/margin": 19.956289291381836, + "fcm_dpo/q_t": 0.38213831186294556, + "grad_norm": 26.93992805480957, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.641958475112915, + "logits/rejected": -0.6092862486839294, + "logps/chosen": -71.08255004882812, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -111.27102661132812, + "loss": 1.0254, + "margin_dpo/margin_mean": 19.95629119873047, + "margin_dpo/margin_std": 24.62198829650879, + "step": 85 + }, + { + "KL/chosen_KL_mean": -16.4979190826416, + "KL/mean": -27.12005615234375, + "KL/rejected_KL_mean": -37.7421875, + "KL/std": 27.8272705078125, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.025832965970039368, + "fcm_dpo/delta": -0.1571456640958786, + "fcm_dpo/margin": 21.244277954101562, + "fcm_dpo/q_t": 0.3934088349342346, + "grad_norm": 31.247526168823242, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6652317047119141, + "logits/rejected": -0.6517903804779053, + "logps/chosen": -72.63538360595703, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -125.86383819580078, + "loss": 1.1045, + "margin_dpo/margin_mean": 21.244277954101562, + "margin_dpo/margin_std": 38.27693176269531, + "step": 86 + }, + { + "KL/chosen_KL_mean": -16.15471649169922, + "KL/mean": -28.62390899658203, + "KL/rejected_KL_mean": -41.09310531616211, + "KL/std": 27.785911560058594, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.02453221194446087, + "fcm_dpo/delta": -0.22750557959079742, + "fcm_dpo/margin": 24.938386917114258, + "fcm_dpo/q_t": 0.3677721619606018, + "grad_norm": 26.31862449645996, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.717815637588501, + "logits/rejected": -0.6921704411506653, + "logps/chosen": -71.79080963134766, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -136.56068420410156, + "loss": 1.0129, + "margin_dpo/margin_mean": 24.93838882446289, + "margin_dpo/margin_std": 33.497528076171875, + "step": 87 + }, + { + "KL/chosen_KL_mean": -20.493343353271484, + "KL/mean": -29.836345672607422, + "KL/rejected_KL_mean": -39.17934799194336, + "KL/std": 28.84607696533203, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.024055175483226776, + "fcm_dpo/delta": -0.05276504531502724, + "fcm_dpo/margin": 18.686004638671875, + "fcm_dpo/q_t": 0.40402984619140625, + "grad_norm": 27.747329711914062, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6649001240730286, + "logits/rejected": -0.6574603319168091, + "logps/chosen": -94.16448974609375, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -145.88784790039062, + "loss": 1.1194, + "margin_dpo/margin_mean": 18.686004638671875, + "margin_dpo/margin_std": 30.319538116455078, + "step": 88 + }, + { + "KL/chosen_KL_mean": -12.558465957641602, + "KL/mean": -23.090679168701172, + "KL/rejected_KL_mean": -33.622894287109375, + "KL/std": 24.5167236328125, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.023851945996284485, + "fcm_dpo/delta": -0.1077832505106926, + "fcm_dpo/margin": 21.064424514770508, + "fcm_dpo/q_t": 0.38852643966674805, + "grad_norm": 25.065860748291016, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7390029430389404, + "logits/rejected": -0.7072830200195312, + "logps/chosen": -73.18338012695312, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -115.7064437866211, + "loss": 1.0405, + "margin_dpo/margin_mean": 21.06442642211914, + "margin_dpo/margin_std": 27.825477600097656, + "step": 89 + }, + { + "KL/chosen_KL_mean": -16.242807388305664, + "KL/mean": -30.51595687866211, + "KL/rejected_KL_mean": -44.78911209106445, + "KL/std": 33.345252990722656, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.02290850505232811, + "fcm_dpo/delta": -0.2710408866405487, + "fcm_dpo/margin": 28.54629898071289, + "fcm_dpo/q_t": 0.3705395460128784, + "grad_norm": 26.39919090270996, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6493753790855408, + "logits/rejected": -0.6614657044410706, + "logps/chosen": -69.52812194824219, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -156.33383178710938, + "loss": 1.0207, + "margin_dpo/margin_mean": 28.54629898071289, + "margin_dpo/margin_std": 41.31671905517578, + "step": 90 + }, + { + "KL/chosen_KL_mean": -16.90831184387207, + "KL/mean": -28.781269073486328, + "KL/rejected_KL_mean": -40.65422821044922, + "KL/std": 26.88111114501953, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.022018849849700928, + "fcm_dpo/delta": -0.12943598628044128, + "fcm_dpo/margin": 23.74591636657715, + "fcm_dpo/q_t": 0.388397753238678, + "grad_norm": 25.34270668029785, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6914452314376831, + "logits/rejected": -0.6580153703689575, + "logps/chosen": -78.71127319335938, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -128.52818298339844, + "loss": 1.0581, + "margin_dpo/margin_mean": 23.745914459228516, + "margin_dpo/margin_std": 34.71296691894531, + "step": 91 + }, + { + "KL/chosen_KL_mean": -15.339115142822266, + "KL/mean": -27.303653717041016, + "KL/rejected_KL_mean": -39.26819610595703, + "KL/std": 27.628177642822266, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.021522948518395424, + "fcm_dpo/delta": -0.12122107297182083, + "fcm_dpo/margin": 23.929075241088867, + "fcm_dpo/q_t": 0.38861674070358276, + "grad_norm": 23.875722885131836, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.710525393486023, + "logits/rejected": -0.6871220469474792, + "logps/chosen": -66.9798812866211, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -117.14938354492188, + "loss": 1.0522, + "margin_dpo/margin_mean": 23.929075241088867, + "margin_dpo/margin_std": 33.46467590332031, + "step": 92 + }, + { + "KL/chosen_KL_mean": -17.250686645507812, + "KL/mean": -29.28143310546875, + "KL/rejected_KL_mean": -41.31217575073242, + "KL/std": 26.182392120361328, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.02094567008316517, + "fcm_dpo/delta": -0.10942681133747101, + "fcm_dpo/margin": 24.06149673461914, + "fcm_dpo/q_t": 0.38804519176483154, + "grad_norm": 24.1500244140625, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7454243898391724, + "logits/rejected": -0.6974157094955444, + "logps/chosen": -69.77992248535156, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -118.4729232788086, + "loss": 1.0293, + "margin_dpo/margin_mean": 24.06149673461914, + "margin_dpo/margin_std": 30.17089080810547, + "step": 93 + }, + { + "KL/chosen_KL_mean": -18.599916458129883, + "KL/mean": -32.482337951660156, + "KL/rejected_KL_mean": -46.3647575378418, + "KL/std": 29.9700984954834, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.020172812044620514, + "fcm_dpo/delta": -0.1712406873703003, + "fcm_dpo/margin": 27.764842987060547, + "fcm_dpo/q_t": 0.37485361099243164, + "grad_norm": 23.93221092224121, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6663378477096558, + "logits/rejected": -0.6508735418319702, + "logps/chosen": -79.82252502441406, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -145.9637908935547, + "loss": 0.9925, + "margin_dpo/margin_mean": 27.764842987060547, + "margin_dpo/margin_std": 31.303754806518555, + "step": 94 + }, + { + "KL/chosen_KL_mean": -17.725191116333008, + "KL/mean": -28.992572784423828, + "KL/rejected_KL_mean": -40.25995635986328, + "KL/std": 28.61754608154297, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.01988731138408184, + "fcm_dpo/delta": -0.050858959555625916, + "fcm_dpo/margin": 22.534767150878906, + "fcm_dpo/q_t": 0.3973836600780487, + "grad_norm": 22.316652297973633, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6923412680625916, + "logits/rejected": -0.6634332537651062, + "logps/chosen": -70.24884033203125, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -116.14030456542969, + "loss": 1.0698, + "margin_dpo/margin_mean": 22.534767150878906, + "margin_dpo/margin_std": 29.937744140625, + "step": 95 + }, + { + "KL/chosen_KL_mean": -17.18556785583496, + "KL/mean": -32.44683837890625, + "KL/rejected_KL_mean": -47.708106994628906, + "KL/std": 32.553123474121094, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.019188063219189644, + "fcm_dpo/delta": -0.19968557357788086, + "fcm_dpo/margin": 30.522541046142578, + "fcm_dpo/q_t": 0.37198513746261597, + "grad_norm": 22.64855194091797, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7047948837280273, + "logits/rejected": -0.6751877069473267, + "logps/chosen": -79.34254455566406, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -144.3041229248047, + "loss": 0.9837, + "margin_dpo/margin_mean": 30.52254295349121, + "margin_dpo/margin_std": 35.89656066894531, + "step": 96 + }, + { + "KL/chosen_KL_mean": -18.515424728393555, + "KL/mean": -30.33550262451172, + "KL/rejected_KL_mean": -42.15557861328125, + "KL/std": 27.115768432617188, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.0191325843334198, + "fcm_dpo/delta": -0.05578317120671272, + "fcm_dpo/margin": 23.640159606933594, + "fcm_dpo/q_t": 0.39513349533081055, + "grad_norm": 23.765064239501953, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6259379386901855, + "logits/rejected": -0.5811424255371094, + "logps/chosen": -73.16178894042969, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -119.12033081054688, + "loss": 1.0533, + "margin_dpo/margin_mean": 23.640159606933594, + "margin_dpo/margin_std": 26.971050262451172, + "step": 97 + }, + { + "KL/chosen_KL_mean": -22.962387084960938, + "KL/mean": -36.437049865722656, + "KL/rejected_KL_mean": -49.911705017089844, + "KL/std": 31.023151397705078, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.018586162477731705, + "fcm_dpo/delta": -0.10644808411598206, + "fcm_dpo/margin": 26.94931983947754, + "fcm_dpo/q_t": 0.3857658803462982, + "grad_norm": 24.38271141052246, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6928247809410095, + "logits/rejected": -0.6772025227546692, + "logps/chosen": -88.22101593017578, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -146.43919372558594, + "loss": 1.058, + "margin_dpo/margin_mean": 26.949317932128906, + "margin_dpo/margin_std": 37.24348449707031, + "step": 98 + }, + { + "KL/chosen_KL_mean": -18.005695343017578, + "KL/mean": -34.60300827026367, + "KL/rejected_KL_mean": -51.2003173828125, + "KL/std": 33.723876953125, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.01790214702486992, + "fcm_dpo/delta": -0.20733481645584106, + "fcm_dpo/margin": 33.19462203979492, + "fcm_dpo/q_t": 0.373191237449646, + "grad_norm": 21.624513626098633, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6868900060653687, + "logits/rejected": -0.6874243021011353, + "logps/chosen": -63.64418029785156, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -137.63824462890625, + "loss": 0.991, + "margin_dpo/margin_mean": 33.19462203979492, + "margin_dpo/margin_std": 41.95561218261719, + "step": 99 + }, + { + "KL/chosen_KL_mean": -20.30302619934082, + "KL/mean": -29.979393005371094, + "KL/rejected_KL_mean": -39.655757904052734, + "KL/std": 27.00857162475586, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.017977459356188774, + "fcm_dpo/delta": 0.05343706160783768, + "fcm_dpo/margin": 19.352737426757812, + "fcm_dpo/q_t": 0.4214937090873718, + "grad_norm": 23.938840866088867, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6565019488334656, + "logits/rejected": -0.6132710576057434, + "logps/chosen": -77.89700317382812, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -113.71597290039062, + "loss": 1.1644, + "margin_dpo/margin_mean": 19.352737426757812, + "margin_dpo/margin_std": 34.764198303222656, + "step": 100 + }, + { + "KL/chosen_KL_mean": -25.042964935302734, + "KL/mean": -36.82512664794922, + "KL/rejected_KL_mean": -48.6072883605957, + "KL/std": 33.41741180419922, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.017799843102693558, + "fcm_dpo/delta": -0.021045971661806107, + "fcm_dpo/margin": 23.564315795898438, + "fcm_dpo/q_t": 0.4105093777179718, + "grad_norm": 23.80852508544922, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6629771590232849, + "logits/rejected": -0.6310428380966187, + "logps/chosen": -86.69181823730469, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -131.79696655273438, + "loss": 1.1272, + "margin_dpo/margin_mean": 23.564315795898438, + "margin_dpo/margin_std": 40.394569396972656, + "step": 101 + }, + { + "KL/chosen_KL_mean": -27.190393447875977, + "KL/mean": -35.897396087646484, + "KL/rejected_KL_mean": -44.604400634765625, + "KL/std": 31.46831703186035, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.01823844388127327, + "fcm_dpo/delta": 0.08336644619703293, + "fcm_dpo/margin": 17.414005279541016, + "fcm_dpo/q_t": 0.4300295412540436, + "grad_norm": 27.354732513427734, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.6968875527381897, + "logits/rejected": -0.6526229977607727, + "logps/chosen": -91.26927185058594, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -112.79147338867188, + "loss": 1.205, + "margin_dpo/margin_mean": 17.414005279541016, + "margin_dpo/margin_std": 37.20274353027344, + "step": 102 + }, + { + "KL/chosen_KL_mean": -21.43701171875, + "KL/mean": -38.73309326171875, + "KL/rejected_KL_mean": -56.029178619384766, + "KL/std": 34.92048645019531, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.017558695748448372, + "fcm_dpo/delta": -0.22132933139801025, + "fcm_dpo/margin": 34.5921630859375, + "fcm_dpo/q_t": 0.3687490224838257, + "grad_norm": 23.33552360534668, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6700901389122009, + "logits/rejected": -0.6359903812408447, + "logps/chosen": -82.73628997802734, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -149.60189819335938, + "loss": 0.9802, + "margin_dpo/margin_mean": 34.5921630859375, + "margin_dpo/margin_std": 42.0259895324707, + "step": 103 + }, + { + "KL/chosen_KL_mean": -23.590349197387695, + "KL/mean": -40.05005645751953, + "KL/rejected_KL_mean": -56.50975799560547, + "KL/std": 38.682777404785156, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.016995321959257126, + "fcm_dpo/delta": -0.16868509352207184, + "fcm_dpo/margin": 32.919410705566406, + "fcm_dpo/q_t": 0.38362085819244385, + "grad_norm": 22.781251907348633, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6920280456542969, + "logits/rejected": -0.6609938144683838, + "logps/chosen": -77.96311950683594, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -146.07447814941406, + "loss": 1.0396, + "margin_dpo/margin_mean": 32.91941452026367, + "margin_dpo/margin_std": 47.22514343261719, + "step": 104 + }, + { + "KL/chosen_KL_mean": -21.948179244995117, + "KL/mean": -45.291290283203125, + "KL/rejected_KL_mean": -68.6343994140625, + "KL/std": 37.714012145996094, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.015931686386466026, + "fcm_dpo/delta": -0.3711584508419037, + "fcm_dpo/margin": 46.686222076416016, + "fcm_dpo/q_t": 0.3327779769897461, + "grad_norm": 22.729717254638672, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.6819756031036377, + "logits/rejected": -0.6596728563308716, + "logps/chosen": -76.58712768554688, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -166.60791015625, + "loss": 0.8676, + "margin_dpo/margin_mean": 46.686222076416016, + "margin_dpo/margin_std": 41.460235595703125, + "step": 105 + }, + { + "KL/chosen_KL_mean": -24.634428024291992, + "KL/mean": -39.239723205566406, + "KL/rejected_KL_mean": -53.84502410888672, + "KL/std": 32.751773834228516, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.015457410365343094, + "fcm_dpo/delta": -0.054143212735652924, + "fcm_dpo/margin": 29.210590362548828, + "fcm_dpo/q_t": 0.3963199853897095, + "grad_norm": 22.359628677368164, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.637825608253479, + "logits/rejected": -0.6066671013832092, + "logps/chosen": -79.46732330322266, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -139.06964111328125, + "loss": 1.0563, + "margin_dpo/margin_mean": 29.210590362548828, + "margin_dpo/margin_std": 35.74064636230469, + "step": 106 + }, + { + "KL/chosen_KL_mean": -29.412578582763672, + "KL/mean": -46.000221252441406, + "KL/rejected_KL_mean": -62.587852478027344, + "KL/std": 40.035980224609375, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.015133179724216461, + "fcm_dpo/delta": -0.10828714072704315, + "fcm_dpo/margin": 33.17527770996094, + "fcm_dpo/q_t": 0.3863827884197235, + "grad_norm": 21.27107810974121, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6799710988998413, + "logits/rejected": -0.6344591379165649, + "logps/chosen": -99.12039184570312, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -157.32736206054688, + "loss": 1.0405, + "margin_dpo/margin_mean": 33.17527770996094, + "margin_dpo/margin_std": 42.199729919433594, + "step": 107 + }, + { + "KL/chosen_KL_mean": -26.358858108520508, + "KL/mean": -46.201507568359375, + "KL/rejected_KL_mean": -66.04415893554688, + "KL/std": 44.01683807373047, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.014687133952975273, + "fcm_dpo/delta": -0.19415885210037231, + "fcm_dpo/margin": 39.685302734375, + "fcm_dpo/q_t": 0.37917497754096985, + "grad_norm": 21.613059997558594, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.6559147834777832, + "logits/rejected": -0.6335631608963013, + "logps/chosen": -82.36874389648438, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -161.84017944335938, + "loss": 1.034, + "margin_dpo/margin_mean": 39.685302734375, + "margin_dpo/margin_std": 56.89977264404297, + "step": 108 + }, + { + "KL/chosen_KL_mean": -24.745630264282227, + "KL/mean": -45.901947021484375, + "KL/rejected_KL_mean": -67.0582504272461, + "KL/std": 42.35730743408203, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.014046456664800644, + "fcm_dpo/delta": -0.2076481282711029, + "fcm_dpo/margin": 42.3126220703125, + "fcm_dpo/q_t": 0.3678189814090729, + "grad_norm": 22.679563522338867, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5965989828109741, + "logits/rejected": -0.5746294260025024, + "logps/chosen": -87.63111877441406, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -165.74398803710938, + "loss": 0.9861, + "margin_dpo/margin_mean": 42.3126220703125, + "margin_dpo/margin_std": 49.79395294189453, + "step": 109 + }, + { + "KL/chosen_KL_mean": -25.548309326171875, + "KL/mean": -44.62482452392578, + "KL/rejected_KL_mean": -63.701332092285156, + "KL/std": 42.32118225097656, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.013520041480660439, + "fcm_dpo/delta": -0.12677742540836334, + "fcm_dpo/margin": 38.15302276611328, + "fcm_dpo/q_t": 0.38655269145965576, + "grad_norm": 19.33060073852539, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.6206210851669312, + "logits/rejected": -0.5850643515586853, + "logps/chosen": -84.30199432373047, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -143.45135498046875, + "loss": 1.0506, + "margin_dpo/margin_mean": 38.15302658081055, + "margin_dpo/margin_std": 50.857452392578125, + "step": 110 + }, + { + "KL/chosen_KL_mean": -29.135221481323242, + "KL/mean": -50.737335205078125, + "KL/rejected_KL_mean": -72.33944702148438, + "KL/std": 47.84112548828125, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.013165132142603397, + "fcm_dpo/delta": -0.18059711158275604, + "fcm_dpo/margin": 43.20423126220703, + "fcm_dpo/q_t": 0.37618499994277954, + "grad_norm": 21.215679168701172, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.6436001062393188, + "logits/rejected": -0.6206039190292358, + "logps/chosen": -97.75932312011719, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -170.768310546875, + "loss": 1.0308, + "margin_dpo/margin_mean": 43.20423126220703, + "margin_dpo/margin_std": 58.755332946777344, + "step": 111 + }, + { + "KL/chosen_KL_mean": -26.79393768310547, + "KL/mean": -40.663818359375, + "KL/rejected_KL_mean": -54.5337028503418, + "KL/std": 33.432289123535156, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.013131228275597095, + "fcm_dpo/delta": 0.0368725061416626, + "fcm_dpo/margin": 27.739761352539062, + "fcm_dpo/q_t": 0.4164872169494629, + "grad_norm": 19.95468521118164, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.6322985887527466, + "logits/rejected": -0.5939322710037231, + "logps/chosen": -77.0435791015625, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -119.30813598632812, + "loss": 1.1223, + "margin_dpo/margin_mean": 27.739761352539062, + "margin_dpo/margin_std": 40.22670364379883, + "step": 112 + }, + { + "KL/chosen_KL_mean": -33.66692352294922, + "KL/mean": -48.52581024169922, + "KL/rejected_KL_mean": -63.38470458984375, + "KL/std": 34.734649658203125, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.013229789212346077, + "fcm_dpo/delta": 0.007117174565792084, + "fcm_dpo/margin": 29.717777252197266, + "fcm_dpo/q_t": 0.40937259793281555, + "grad_norm": 20.3212947845459, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.6194804906845093, + "logits/rejected": -0.5778924226760864, + "logps/chosen": -100.37987518310547, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -141.35340881347656, + "loss": 1.0918, + "margin_dpo/margin_mean": 29.717777252197266, + "margin_dpo/margin_std": 38.455169677734375, + "step": 113 + }, + { + "KL/chosen_KL_mean": -31.021665573120117, + "KL/mean": -54.70832061767578, + "KL/rejected_KL_mean": -78.39497375488281, + "KL/std": 52.318634033203125, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.012726202607154846, + "fcm_dpo/delta": -0.21821393072605133, + "fcm_dpo/margin": 47.373313903808594, + "fcm_dpo/q_t": 0.3736953139305115, + "grad_norm": 21.744464874267578, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.6348057985305786, + "logits/rejected": -0.6047931909561157, + "logps/chosen": -88.80674743652344, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -165.504638671875, + "loss": 0.9973, + "margin_dpo/margin_mean": 47.373313903808594, + "margin_dpo/margin_std": 63.302581787109375, + "step": 114 + }, + { + "KL/chosen_KL_mean": -40.448272705078125, + "KL/mean": -56.732337951660156, + "KL/rejected_KL_mean": -73.01640319824219, + "KL/std": 49.903263092041016, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.01267709955573082, + "fcm_dpo/delta": -0.013551291078329086, + "fcm_dpo/margin": 32.56813049316406, + "fcm_dpo/q_t": 0.41360044479370117, + "grad_norm": 25.791486740112305, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.65025395154953, + "logits/rejected": -0.6308355331420898, + "logps/chosen": -106.03091430664062, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -171.58192443847656, + "loss": 1.1635, + "margin_dpo/margin_mean": 32.56813049316406, + "margin_dpo/margin_std": 65.4981918334961, + "step": 115 + }, + { + "KL/chosen_KL_mean": -30.46641731262207, + "KL/mean": -48.50792694091797, + "KL/rejected_KL_mean": -66.5494384765625, + "KL/std": 42.040828704833984, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.012582490220665932, + "fcm_dpo/delta": -0.05673955753445625, + "fcm_dpo/margin": 36.08301544189453, + "fcm_dpo/q_t": 0.3975168466567993, + "grad_norm": 22.044328689575195, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.5849428176879883, + "logits/rejected": -0.5729961395263672, + "logps/chosen": -81.86672973632812, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -147.0712890625, + "loss": 1.0575, + "margin_dpo/margin_mean": 36.08301544189453, + "margin_dpo/margin_std": 45.310997009277344, + "step": 116 + }, + { + "KL/chosen_KL_mean": -38.867767333984375, + "KL/mean": -53.734153747558594, + "KL/rejected_KL_mean": -68.60052490234375, + "KL/std": 44.917259216308594, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.012545032426714897, + "fcm_dpo/delta": 0.028039565309882164, + "fcm_dpo/margin": 29.732759475708008, + "fcm_dpo/q_t": 0.41659021377563477, + "grad_norm": 27.598058700561523, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.6111999154090881, + "logits/rejected": -0.5768216848373413, + "logps/chosen": -108.16617584228516, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -135.18450927734375, + "loss": 1.1409, + "margin_dpo/margin_mean": 29.732757568359375, + "margin_dpo/margin_std": 51.03728485107422, + "step": 117 + }, + { + "KL/chosen_KL_mean": -30.890750885009766, + "KL/mean": -48.92853546142578, + "KL/rejected_KL_mean": -66.96631622314453, + "KL/std": 40.681114196777344, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.012470767833292484, + "fcm_dpo/delta": -0.052215058356523514, + "fcm_dpo/margin": 36.07556915283203, + "fcm_dpo/q_t": 0.398837149143219, + "grad_norm": 21.093475341796875, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.6295737028121948, + "logits/rejected": -0.610241174697876, + "logps/chosen": -86.53173065185547, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -142.6353759765625, + "loss": 1.0619, + "margin_dpo/margin_mean": 36.07556915283203, + "margin_dpo/margin_std": 46.53681182861328, + "step": 118 + }, + { + "KL/chosen_KL_mean": -42.682559967041016, + "KL/mean": -60.4480094909668, + "KL/rejected_KL_mean": -78.21345520019531, + "KL/std": 45.67112350463867, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.012276040390133858, + "fcm_dpo/delta": -0.03898315131664276, + "fcm_dpo/margin": 35.53089141845703, + "fcm_dpo/q_t": 0.4051964282989502, + "grad_norm": 23.14076042175293, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.6412409543991089, + "logits/rejected": -0.6299257278442383, + "logps/chosen": -116.19275665283203, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -181.19073486328125, + "loss": 1.1022, + "margin_dpo/margin_mean": 35.5308952331543, + "margin_dpo/margin_std": 55.40949630737305, + "step": 119 + }, + { + "KL/chosen_KL_mean": -43.792686462402344, + "KL/mean": -68.31946563720703, + "KL/rejected_KL_mean": -92.84623718261719, + "KL/std": 56.9885368347168, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.011964422650635242, + "fcm_dpo/delta": -0.19873662292957306, + "fcm_dpo/margin": 49.053550720214844, + "fcm_dpo/q_t": 0.3713051676750183, + "grad_norm": 22.053565979003906, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.6161653399467468, + "logits/rejected": -0.5831862688064575, + "logps/chosen": -120.57351684570312, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -200.86997985839844, + "loss": 0.9941, + "margin_dpo/margin_mean": 49.053550720214844, + "margin_dpo/margin_std": 59.88517379760742, + "step": 120 + }, + { + "KL/chosen_KL_mean": -42.266502380371094, + "KL/mean": -69.31425476074219, + "KL/rejected_KL_mean": -96.36199951171875, + "KL/std": 55.65351867675781, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.011389784514904022, + "fcm_dpo/delta": -0.23216752707958221, + "fcm_dpo/margin": 54.09550476074219, + "fcm_dpo/q_t": 0.3677094280719757, + "grad_norm": 24.86897087097168, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5836307406425476, + "logits/rejected": -0.5795871019363403, + "logps/chosen": -104.056396484375, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -206.3565673828125, + "loss": 0.9972, + "margin_dpo/margin_mean": 54.09550476074219, + "margin_dpo/margin_std": 69.78767395019531, + "step": 121 + }, + { + "KL/chosen_KL_mean": -38.54806137084961, + "KL/mean": -72.24299621582031, + "KL/rejected_KL_mean": -105.93792724609375, + "KL/std": 63.453697204589844, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.010750418528914452, + "fcm_dpo/delta": -0.34964752197265625, + "fcm_dpo/margin": 67.3898696899414, + "fcm_dpo/q_t": 0.3409060835838318, + "grad_norm": 23.406606674194336, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.5471662282943726, + "logits/rejected": -0.5631359815597534, + "logps/chosen": -85.45027160644531, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -212.65213012695312, + "loss": 0.8947, + "margin_dpo/margin_mean": 67.3898696899414, + "margin_dpo/margin_std": 66.38093566894531, + "step": 122 + }, + { + "KL/chosen_KL_mean": -42.40296173095703, + "KL/mean": -64.52996063232422, + "KL/rejected_KL_mean": -86.65696716308594, + "KL/std": 54.35682678222656, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.010394008830189705, + "fcm_dpo/delta": -0.06381888687610626, + "fcm_dpo/margin": 44.254005432128906, + "fcm_dpo/q_t": 0.4011934697628021, + "grad_norm": 20.68321418762207, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.5610964298248291, + "logits/rejected": -0.5440037250518799, + "logps/chosen": -103.7416000366211, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -174.432373046875, + "loss": 1.1072, + "margin_dpo/margin_mean": 44.254005432128906, + "margin_dpo/margin_std": 72.87324523925781, + "step": 123 + }, + { + "KL/chosen_KL_mean": -48.31520462036133, + "KL/mean": -81.04126739501953, + "KL/rejected_KL_mean": -113.76734161376953, + "KL/std": 74.36558532714844, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.009977076202630997, + "fcm_dpo/delta": -0.2716488838195801, + "fcm_dpo/margin": 65.4521255493164, + "fcm_dpo/q_t": 0.36892297863960266, + "grad_norm": 22.609094619750977, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.5452510714530945, + "logits/rejected": -0.5474103689193726, + "logps/chosen": -119.76353454589844, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -231.347900390625, + "loss": 1.0131, + "margin_dpo/margin_mean": 65.4521255493164, + "margin_dpo/margin_std": 93.51290130615234, + "step": 124 + }, + { + "KL/chosen_KL_mean": -39.86096954345703, + "KL/mean": -68.74577331542969, + "KL/rejected_KL_mean": -97.63056945800781, + "KL/std": 62.61181640625, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.009643394500017166, + "fcm_dpo/delta": -0.1661270409822464, + "fcm_dpo/margin": 57.76960754394531, + "fcm_dpo/q_t": 0.38036981225013733, + "grad_norm": 18.92750358581543, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.5110180377960205, + "logits/rejected": -0.5126426219940186, + "logps/chosen": -89.99790954589844, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -181.6191864013672, + "loss": 1.0277, + "margin_dpo/margin_mean": 57.76960754394531, + "margin_dpo/margin_std": 77.59306335449219, + "step": 125 + }, + { + "KL/chosen_KL_mean": -43.15443420410156, + "KL/mean": -70.15377807617188, + "KL/rejected_KL_mean": -97.15312194824219, + "KL/std": 56.47283935546875, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.009367674589157104, + "fcm_dpo/delta": -0.11177867650985718, + "fcm_dpo/margin": 53.99868392944336, + "fcm_dpo/q_t": 0.3888355791568756, + "grad_norm": 20.72877311706543, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.5301386713981628, + "logits/rejected": -0.5246546268463135, + "logps/chosen": -98.82150268554688, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -195.2828826904297, + "loss": 1.0427, + "margin_dpo/margin_mean": 53.998687744140625, + "margin_dpo/margin_std": 71.67208099365234, + "step": 126 + }, + { + "KL/chosen_KL_mean": -41.97722244262695, + "KL/mean": -61.81834411621094, + "KL/rejected_KL_mean": -81.65946960449219, + "KL/std": 49.690757751464844, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.009377237409353256, + "fcm_dpo/delta": 0.02894909866154194, + "fcm_dpo/margin": 39.68224334716797, + "fcm_dpo/q_t": 0.41414302587509155, + "grad_norm": 21.78166389465332, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.5337643623352051, + "logits/rejected": -0.5144004821777344, + "logps/chosen": -98.53190612792969, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -158.4552459716797, + "loss": 1.1295, + "margin_dpo/margin_mean": 39.68224334716797, + "margin_dpo/margin_std": 62.61986541748047, + "step": 127 + }, + { + "KL/chosen_KL_mean": -44.89812469482422, + "KL/mean": -65.68916320800781, + "KL/rejected_KL_mean": -86.48019409179688, + "KL/std": 58.180091857910156, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.009436184540390968, + "fcm_dpo/delta": 0.007552694529294968, + "fcm_dpo/margin": 41.58207702636719, + "fcm_dpo/q_t": 0.41283971071243286, + "grad_norm": 29.683269500732422, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.5333284139633179, + "logits/rejected": -0.5138853788375854, + "logps/chosen": -103.01908111572266, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -162.91915893554688, + "loss": 1.1417, + "margin_dpo/margin_mean": 41.58207321166992, + "margin_dpo/margin_std": 72.79296112060547, + "step": 128 + }, + { + "KL/chosen_KL_mean": -54.7447624206543, + "KL/mean": -76.73641967773438, + "KL/rejected_KL_mean": -98.72806549072266, + "KL/std": 64.67262268066406, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.009391024708747864, + "fcm_dpo/delta": -0.013617277145385742, + "fcm_dpo/margin": 43.983299255371094, + "fcm_dpo/q_t": 0.4138565957546234, + "grad_norm": 21.114395141601562, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.552919864654541, + "logits/rejected": -0.5430943369865417, + "logps/chosen": -121.6611328125, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -195.37026977539062, + "loss": 1.1488, + "margin_dpo/margin_mean": 43.98330307006836, + "margin_dpo/margin_std": 84.16700744628906, + "step": 129 + }, + { + "KL/chosen_KL_mean": -40.41427993774414, + "KL/mean": -73.63245391845703, + "KL/rejected_KL_mean": -106.85063934326172, + "KL/std": 66.54023742675781, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.009167741052806377, + "fcm_dpo/delta": -0.222553089261055, + "fcm_dpo/margin": 66.43634796142578, + "fcm_dpo/q_t": 0.369380921125412, + "grad_norm": 21.047300338745117, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.5099730491638184, + "logits/rejected": -0.5048457384109497, + "logps/chosen": -85.08113098144531, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -189.63229370117188, + "loss": 0.9955, + "margin_dpo/margin_mean": 66.43634796142578, + "margin_dpo/margin_std": 81.0242691040039, + "step": 130 + }, + { + "KL/chosen_KL_mean": -35.863426208496094, + "KL/mean": -69.09681701660156, + "KL/rejected_KL_mean": -102.33018493652344, + "KL/std": 65.71382141113281, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.008704297244548798, + "fcm_dpo/delta": -0.189858540892601, + "fcm_dpo/margin": 66.46675872802734, + "fcm_dpo/q_t": 0.3684391677379608, + "grad_norm": 29.34795570373535, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.47968876361846924, + "logits/rejected": -0.48612451553344727, + "logps/chosen": -80.78801727294922, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -190.77420043945312, + "loss": 0.9779, + "margin_dpo/margin_mean": 66.46676635742188, + "margin_dpo/margin_std": 71.93128204345703, + "step": 131 + }, + { + "KL/chosen_KL_mean": -48.00788116455078, + "KL/mean": -75.32115936279297, + "KL/rejected_KL_mean": -102.63444519042969, + "KL/std": 65.83612823486328, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.008527089841663837, + "fcm_dpo/delta": -0.06924474984407425, + "fcm_dpo/margin": 54.626564025878906, + "fcm_dpo/q_t": 0.4006684422492981, + "grad_norm": 20.385377883911133, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.5199368000030518, + "logits/rejected": -0.5177985429763794, + "logps/chosen": -107.00896453857422, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -190.52659606933594, + "loss": 1.0865, + "margin_dpo/margin_mean": 54.626564025878906, + "margin_dpo/margin_std": 85.50218200683594, + "step": 132 + }, + { + "KL/chosen_KL_mean": -56.399330139160156, + "KL/mean": -80.62858581542969, + "KL/rejected_KL_mean": -104.85784912109375, + "KL/std": 59.521888732910156, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.008482728153467178, + "fcm_dpo/delta": -0.01163606159389019, + "fcm_dpo/margin": 48.45851516723633, + "fcm_dpo/q_t": 0.40974992513656616, + "grad_norm": 27.659805297851562, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.48731863498687744, + "logits/rejected": -0.47168076038360596, + "logps/chosen": -123.00382995605469, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -201.19140625, + "loss": 1.1078, + "margin_dpo/margin_mean": 48.45851135253906, + "margin_dpo/margin_std": 75.3657455444336, + "step": 133 + }, + { + "KL/chosen_KL_mean": -45.17230224609375, + "KL/mean": -71.02302551269531, + "KL/rejected_KL_mean": -96.87374877929688, + "KL/std": 58.303531646728516, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.008472021669149399, + "fcm_dpo/delta": -0.039847277104854584, + "fcm_dpo/margin": 51.701446533203125, + "fcm_dpo/q_t": 0.4017735719680786, + "grad_norm": 18.833871841430664, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.48723411560058594, + "logits/rejected": -0.47374820709228516, + "logps/chosen": -97.24156188964844, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -184.52825927734375, + "loss": 1.0672, + "margin_dpo/margin_mean": 51.701446533203125, + "margin_dpo/margin_std": 67.38398742675781, + "step": 134 + }, + { + "KL/chosen_KL_mean": -49.796905517578125, + "KL/mean": -86.9676513671875, + "KL/rejected_KL_mean": -124.13839721679688, + "KL/std": 77.2548828125, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.008155114948749542, + "fcm_dpo/delta": -0.2196299433708191, + "fcm_dpo/margin": 74.34149169921875, + "fcm_dpo/q_t": 0.37047868967056274, + "grad_norm": 22.309160232543945, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.49530279636383057, + "logits/rejected": -0.5288498997688293, + "logps/chosen": -100.15076446533203, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -240.11814880371094, + "loss": 0.9937, + "margin_dpo/margin_mean": 74.34149169921875, + "margin_dpo/margin_std": 93.30258178710938, + "step": 135 + }, + { + "KL/chosen_KL_mean": -59.11945343017578, + "KL/mean": -83.80839538574219, + "KL/rejected_KL_mean": -108.49734497070312, + "KL/std": 69.72404479980469, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.008020879700779915, + "fcm_dpo/delta": 0.0033986419439315796, + "fcm_dpo/margin": 49.377891540527344, + "fcm_dpo/q_t": 0.41878652572631836, + "grad_norm": 20.5084285736084, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.5011118650436401, + "logits/rejected": -0.4934838116168976, + "logps/chosen": -124.19196319580078, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -204.81857299804688, + "loss": 1.1419, + "margin_dpo/margin_mean": 49.377891540527344, + "margin_dpo/margin_std": 89.17582702636719, + "step": 136 + }, + { + "KL/chosen_KL_mean": -57.257904052734375, + "KL/mean": -96.05001831054688, + "KL/rejected_KL_mean": -134.84214782714844, + "KL/std": 92.76973724365234, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.007803335785865784, + "fcm_dpo/delta": -0.21894118189811707, + "fcm_dpo/margin": 77.58424377441406, + "fcm_dpo/q_t": 0.3764541745185852, + "grad_norm": 18.939228057861328, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.4312455654144287, + "logits/rejected": -0.4585828185081482, + "logps/chosen": -106.01701354980469, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -248.70591735839844, + "loss": 1.0242, + "margin_dpo/margin_mean": 77.58424377441406, + "margin_dpo/margin_std": 110.91220092773438, + "step": 137 + }, + { + "KL/chosen_KL_mean": -59.9859733581543, + "KL/mean": -89.45449829101562, + "KL/rejected_KL_mean": -118.92300415039062, + "KL/std": 71.05049133300781, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.0076743680983781815, + "fcm_dpo/delta": -0.054747480899095535, + "fcm_dpo/margin": 58.93703842163086, + "fcm_dpo/q_t": 0.396518349647522, + "grad_norm": 21.258026123046875, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.46026408672332764, + "logits/rejected": -0.45029088854789734, + "logps/chosen": -120.50562286376953, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -212.11996459960938, + "loss": 1.056, + "margin_dpo/margin_mean": 58.93703842163086, + "margin_dpo/margin_std": 72.72451782226562, + "step": 138 + }, + { + "KL/chosen_KL_mean": -50.09823226928711, + "KL/mean": -85.1839599609375, + "KL/rejected_KL_mean": -120.2696762084961, + "KL/std": 67.29483032226562, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.007506782189011574, + "fcm_dpo/delta": -0.13361655175685883, + "fcm_dpo/margin": 70.17144012451172, + "fcm_dpo/q_t": 0.3833360970020294, + "grad_norm": 18.251789093017578, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.43976572155952454, + "logits/rejected": -0.42997294664382935, + "logps/chosen": -96.98961639404297, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -199.99765014648438, + "loss": 1.0203, + "margin_dpo/margin_mean": 70.17144775390625, + "margin_dpo/margin_std": 87.33370971679688, + "step": 139 + }, + { + "KL/chosen_KL_mean": -60.703773498535156, + "KL/mean": -92.85005187988281, + "KL/rejected_KL_mean": -124.996337890625, + "KL/std": 75.18809509277344, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.007350086234509945, + "fcm_dpo/delta": -0.07621538639068604, + "fcm_dpo/margin": 64.29254913330078, + "fcm_dpo/q_t": 0.3951640725135803, + "grad_norm": 21.289108276367188, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.43051451444625854, + "logits/rejected": -0.41661226749420166, + "logps/chosen": -119.6784896850586, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -208.2804412841797, + "loss": 1.0674, + "margin_dpo/margin_mean": 64.29255676269531, + "margin_dpo/margin_std": 90.5318603515625, + "step": 140 + }, + { + "KL/chosen_KL_mean": -68.46115112304688, + "KL/mean": -98.69406127929688, + "KL/rejected_KL_mean": -128.92697143554688, + "KL/std": 82.50093078613281, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.0072875237092375755, + "fcm_dpo/delta": -0.04249750077724457, + "fcm_dpo/margin": 60.46580123901367, + "fcm_dpo/q_t": 0.40113985538482666, + "grad_norm": 27.49009132385254, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.42894911766052246, + "logits/rejected": -0.4162313938140869, + "logps/chosen": -143.5368194580078, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -227.11923217773438, + "loss": 1.1018, + "margin_dpo/margin_mean": 60.46580123901367, + "margin_dpo/margin_std": 95.05332946777344, + "step": 141 + }, + { + "KL/chosen_KL_mean": -69.84091186523438, + "KL/mean": -104.2848129272461, + "KL/rejected_KL_mean": -138.7287139892578, + "KL/std": 90.0689926147461, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.0071883308701217175, + "fcm_dpo/delta": -0.10009808838367462, + "fcm_dpo/margin": 68.88780975341797, + "fcm_dpo/q_t": 0.3931783437728882, + "grad_norm": 27.38943862915039, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.38956135511398315, + "logits/rejected": -0.3877231776714325, + "logps/chosen": -127.86884307861328, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -233.31094360351562, + "loss": 1.0861, + "margin_dpo/margin_mean": 68.88780975341797, + "margin_dpo/margin_std": 106.31938934326172, + "step": 142 + }, + { + "KL/chosen_KL_mean": -74.0807113647461, + "KL/mean": -96.12844848632812, + "KL/rejected_KL_mean": -118.1761703491211, + "KL/std": 79.65106201171875, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.007182758301496506, + "fcm_dpo/delta": 0.08600388467311859, + "fcm_dpo/margin": 44.095462799072266, + "fcm_dpo/q_t": 0.4324319660663605, + "grad_norm": 23.738691329956055, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.3982440233230591, + "logits/rejected": -0.3918241858482361, + "logps/chosen": -131.67715454101562, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -197.17575073242188, + "loss": 1.1979, + "margin_dpo/margin_mean": 44.095458984375, + "margin_dpo/margin_std": 92.97320556640625, + "step": 143 + }, + { + "KL/chosen_KL_mean": -65.40249633789062, + "KL/mean": -92.5687026977539, + "KL/rejected_KL_mean": -119.73490905761719, + "KL/std": 67.10587310791016, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.0072448779828846455, + "fcm_dpo/delta": 0.006620034109801054, + "fcm_dpo/margin": 54.33241271972656, + "fcm_dpo/q_t": 0.4110024869441986, + "grad_norm": 21.250240325927734, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.3962482213973999, + "logits/rejected": -0.38264691829681396, + "logps/chosen": -125.30885314941406, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -201.73516845703125, + "loss": 1.1073, + "margin_dpo/margin_mean": 54.33241271972656, + "margin_dpo/margin_std": 78.96994018554688, + "step": 144 + }, + { + "KL/chosen_KL_mean": -62.36488342285156, + "KL/mean": -91.61642456054688, + "KL/rejected_KL_mean": -120.86798095703125, + "KL/std": 67.04849243164062, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.007223756983876228, + "fcm_dpo/delta": -0.023631099611520767, + "fcm_dpo/margin": 58.50310516357422, + "fcm_dpo/q_t": 0.40419164299964905, + "grad_norm": 24.44662857055664, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.43467003107070923, + "logits/rejected": -0.41884565353393555, + "logps/chosen": -118.96554565429688, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -198.73428344726562, + "loss": 1.0871, + "margin_dpo/margin_mean": 58.50310516357422, + "margin_dpo/margin_std": 81.39432525634766, + "step": 145 + }, + { + "KL/chosen_KL_mean": -85.68614196777344, + "KL/mean": -108.33212280273438, + "KL/rejected_KL_mean": -130.9781036376953, + "KL/std": 72.9786148071289, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.0072767892852425575, + "fcm_dpo/delta": 0.0728573128581047, + "fcm_dpo/margin": 45.291969299316406, + "fcm_dpo/q_t": 0.42524510622024536, + "grad_norm": 26.622976303100586, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.40609169006347656, + "logits/rejected": -0.3791778087615967, + "logps/chosen": -151.68658447265625, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -212.680908203125, + "loss": 1.1829, + "margin_dpo/margin_mean": 45.291969299316406, + "margin_dpo/margin_std": 88.1439437866211, + "step": 146 + }, + { + "KL/chosen_KL_mean": -62.47882843017578, + "KL/mean": -93.27926635742188, + "KL/rejected_KL_mean": -124.0796890258789, + "KL/std": 73.8756103515625, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.007257610559463501, + "fcm_dpo/delta": -0.04944849759340286, + "fcm_dpo/margin": 61.60086441040039, + "fcm_dpo/q_t": 0.40133196115493774, + "grad_norm": 19.749757766723633, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.44524866342544556, + "logits/rejected": -0.4258359670639038, + "logps/chosen": -115.88431549072266, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -195.47030639648438, + "loss": 1.0884, + "margin_dpo/margin_mean": 61.60086441040039, + "margin_dpo/margin_std": 91.78453063964844, + "step": 147 + }, + { + "KL/chosen_KL_mean": -61.59424591064453, + "KL/mean": -88.01561737060547, + "KL/rejected_KL_mean": -114.43699645996094, + "KL/std": 73.69937133789062, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.007336227223277092, + "fcm_dpo/delta": 0.010792955756187439, + "fcm_dpo/margin": 52.84275436401367, + "fcm_dpo/q_t": 0.41496026515960693, + "grad_norm": 19.450883865356445, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.4953920841217041, + "logits/rejected": -0.48874759674072266, + "logps/chosen": -126.53132629394531, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -217.53085327148438, + "loss": 1.1198, + "margin_dpo/margin_mean": 52.842750549316406, + "margin_dpo/margin_std": 77.52517700195312, + "step": 148 + }, + { + "KL/chosen_KL_mean": -58.77273178100586, + "KL/mean": -91.42341613769531, + "KL/rejected_KL_mean": -124.0740966796875, + "KL/std": 66.24546813964844, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.007133619859814644, + "fcm_dpo/delta": -0.07102078199386597, + "fcm_dpo/margin": 65.30136108398438, + "fcm_dpo/q_t": 0.393667072057724, + "grad_norm": 18.487836837768555, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.4818879961967468, + "logits/rejected": -0.466863214969635, + "logps/chosen": -117.24649047851562, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -223.3888397216797, + "loss": 1.0514, + "margin_dpo/margin_mean": 65.30136108398438, + "margin_dpo/margin_std": 78.66407775878906, + "step": 149 + }, + { + "KL/chosen_KL_mean": -51.80621337890625, + "KL/mean": -83.86331176757812, + "KL/rejected_KL_mean": -115.92042541503906, + "KL/std": 77.99642944335938, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.007055122405290604, + "fcm_dpo/delta": -0.05630442500114441, + "fcm_dpo/margin": 64.11421203613281, + "fcm_dpo/q_t": 0.4028938412666321, + "grad_norm": 17.89496612548828, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.4605367183685303, + "logits/rejected": -0.4642760455608368, + "logps/chosen": -97.51202392578125, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -199.2680206298828, + "loss": 1.0778, + "margin_dpo/margin_mean": 64.11421203613281, + "margin_dpo/margin_std": 93.35562896728516, + "step": 150 + }, + { + "KL/chosen_KL_mean": -64.34500122070312, + "KL/mean": -96.52954864501953, + "KL/rejected_KL_mean": -128.71409606933594, + "KL/std": 73.31095123291016, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.007025968283414841, + "fcm_dpo/delta": -0.05475372448563576, + "fcm_dpo/margin": 64.36907958984375, + "fcm_dpo/q_t": 0.39651280641555786, + "grad_norm": 20.133039474487305, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.5430213809013367, + "logits/rejected": -0.5170940160751343, + "logps/chosen": -134.91583251953125, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -229.17791748046875, + "loss": 1.0571, + "margin_dpo/margin_mean": 64.36907958984375, + "margin_dpo/margin_std": 80.22126770019531, + "step": 151 + }, + { + "KL/chosen_KL_mean": -56.113224029541016, + "KL/mean": -95.1072006225586, + "KL/rejected_KL_mean": -134.10118103027344, + "KL/std": 75.43019104003906, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.006877364590764046, + "fcm_dpo/delta": -0.143853098154068, + "fcm_dpo/margin": 77.98796081542969, + "fcm_dpo/q_t": 0.3808814287185669, + "grad_norm": 20.090179443359375, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.5030619502067566, + "logits/rejected": -0.5021189451217651, + "logps/chosen": -116.2776107788086, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -240.24163818359375, + "loss": 1.0136, + "margin_dpo/margin_mean": 77.98796081542969, + "margin_dpo/margin_std": 92.31379699707031, + "step": 152 + }, + { + "KL/chosen_KL_mean": -57.18572998046875, + "KL/mean": -90.3531265258789, + "KL/rejected_KL_mean": -123.5205307006836, + "KL/std": 81.89944458007812, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.006771269254386425, + "fcm_dpo/delta": -0.051460087299346924, + "fcm_dpo/margin": 66.33480834960938, + "fcm_dpo/q_t": 0.4025265574455261, + "grad_norm": 15.598538398742676, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.4679190218448639, + "logits/rejected": -0.4620906114578247, + "logps/chosen": -113.50100708007812, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -209.17636108398438, + "loss": 1.0873, + "margin_dpo/margin_mean": 66.33480834960938, + "margin_dpo/margin_std": 100.26518249511719, + "step": 153 + }, + { + "KL/chosen_KL_mean": -68.28947448730469, + "KL/mean": -100.119873046875, + "KL/rejected_KL_mean": -131.9502716064453, + "KL/std": 82.33322143554688, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.0067377034574747086, + "fcm_dpo/delta": -0.030444884672760963, + "fcm_dpo/margin": 63.660789489746094, + "fcm_dpo/q_t": 0.4058271646499634, + "grad_norm": 19.02754783630371, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.49149090051651, + "logits/rejected": -0.4947656989097595, + "logps/chosen": -131.03204345703125, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -236.1944580078125, + "loss": 1.1177, + "margin_dpo/margin_mean": 63.660789489746094, + "margin_dpo/margin_std": 104.63356018066406, + "step": 154 + }, + { + "KL/chosen_KL_mean": -62.5084228515625, + "KL/mean": -94.71826171875, + "KL/rejected_KL_mean": -126.9281005859375, + "KL/std": 76.50324249267578, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.0066858758218586445, + "fcm_dpo/delta": -0.03214951232075691, + "fcm_dpo/margin": 64.41968536376953, + "fcm_dpo/q_t": 0.4033392667770386, + "grad_norm": 19.087644577026367, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.47978729009628296, + "logits/rejected": -0.46058785915374756, + "logps/chosen": -123.16160583496094, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -204.42030334472656, + "loss": 1.0916, + "margin_dpo/margin_mean": 64.41968536376953, + "margin_dpo/margin_std": 94.55856323242188, + "step": 155 + }, + { + "KL/chosen_KL_mean": -87.33506774902344, + "KL/mean": -105.73838806152344, + "KL/rejected_KL_mean": -124.14170837402344, + "KL/std": 81.15922546386719, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.006795594468712807, + "fcm_dpo/delta": 0.15397340059280396, + "fcm_dpo/margin": 36.806644439697266, + "fcm_dpo/q_t": 0.44396722316741943, + "grad_norm": 28.234712600708008, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.41929933428764343, + "logits/rejected": -0.41165250539779663, + "logps/chosen": -156.82693481445312, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -201.31100463867188, + "loss": 1.2752, + "margin_dpo/margin_mean": 36.806640625, + "margin_dpo/margin_std": 106.97114562988281, + "step": 156 + }, + { + "KL/chosen_KL_mean": -78.56407928466797, + "KL/mean": -119.37489318847656, + "KL/rejected_KL_mean": -160.18569946289062, + "KL/std": 88.56603240966797, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.0066710347309708595, + "fcm_dpo/delta": -0.15436424314975739, + "fcm_dpo/margin": 81.62162780761719, + "fcm_dpo/q_t": 0.37616726756095886, + "grad_norm": 24.163137435913086, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.3862057626247406, + "logits/rejected": -0.39289337396621704, + "logps/chosen": -139.93251037597656, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -267.83209228515625, + "loss": 1.0227, + "margin_dpo/margin_mean": 81.62162780761719, + "margin_dpo/margin_std": 100.67048645019531, + "step": 157 + }, + { + "KL/chosen_KL_mean": -79.81919860839844, + "KL/mean": -123.40052795410156, + "KL/rejected_KL_mean": -166.98184204101562, + "KL/std": 110.20083618164062, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.006485756486654282, + "fcm_dpo/delta": -0.17520901560783386, + "fcm_dpo/margin": 87.16267395019531, + "fcm_dpo/q_t": 0.3847208619117737, + "grad_norm": 19.84897232055664, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.38193219900131226, + "logits/rejected": -0.39960193634033203, + "logps/chosen": -137.43211364746094, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -280.676513671875, + "loss": 1.0511, + "margin_dpo/margin_mean": 87.16267395019531, + "margin_dpo/margin_std": 132.95082092285156, + "step": 158 + }, + { + "KL/chosen_KL_mean": -89.08233642578125, + "KL/mean": -118.92990112304688, + "KL/rejected_KL_mean": -148.77745056152344, + "KL/std": 95.99357604980469, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.006469495594501495, + "fcm_dpo/delta": 0.01387358084321022, + "fcm_dpo/margin": 59.695098876953125, + "fcm_dpo/q_t": 0.4142230451107025, + "grad_norm": 21.900203704833984, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.43366947770118713, + "logits/rejected": -0.40542280673980713, + "logps/chosen": -170.64268493652344, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -237.6761474609375, + "loss": 1.1438, + "margin_dpo/margin_mean": 59.695098876953125, + "margin_dpo/margin_std": 104.25253295898438, + "step": 159 + }, + { + "KL/chosen_KL_mean": -91.4988021850586, + "KL/mean": -131.28298950195312, + "KL/rejected_KL_mean": -171.0671844482422, + "KL/std": 102.83206176757812, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.006312145851552486, + "fcm_dpo/delta": -0.10859732329845428, + "fcm_dpo/margin": 79.56837463378906, + "fcm_dpo/q_t": 0.3950725197792053, + "grad_norm": 24.09662437438965, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.3782558739185333, + "logits/rejected": -0.3698764443397522, + "logps/chosen": -157.22967529296875, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -268.2850036621094, + "loss": 1.0892, + "margin_dpo/margin_mean": 79.56837463378906, + "margin_dpo/margin_std": 128.68138122558594, + "step": 160 + }, + { + "KL/chosen_KL_mean": -79.04899597167969, + "KL/mean": -113.90074920654297, + "KL/rejected_KL_mean": -148.75250244140625, + "KL/std": 81.52310180664062, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.006268530152738094, + "fcm_dpo/delta": -0.038637034595012665, + "fcm_dpo/margin": 69.70352172851562, + "fcm_dpo/q_t": 0.4036606550216675, + "grad_norm": 22.446317672729492, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.3617369830608368, + "logits/rejected": -0.36027175188064575, + "logps/chosen": -131.48545837402344, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -232.1834716796875, + "loss": 1.0915, + "margin_dpo/margin_mean": 69.70352172851562, + "margin_dpo/margin_std": 103.23650360107422, + "step": 161 + }, + { + "KL/chosen_KL_mean": -75.72311401367188, + "KL/mean": -110.44548034667969, + "KL/rejected_KL_mean": -145.1678466796875, + "KL/std": 90.16195678710938, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.006175653077661991, + "fcm_dpo/delta": -0.03189679980278015, + "fcm_dpo/margin": 69.44474029541016, + "fcm_dpo/q_t": 0.4056203365325928, + "grad_norm": 22.5521240234375, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.42605894804000854, + "logits/rejected": -0.40921443700790405, + "logps/chosen": -138.3336944580078, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -234.5584259033203, + "loss": 1.1056, + "margin_dpo/margin_mean": 69.44474029541016, + "margin_dpo/margin_std": 106.96495056152344, + "step": 162 + }, + { + "KL/chosen_KL_mean": -85.05548095703125, + "KL/mean": -118.73371887207031, + "KL/rejected_KL_mean": -152.41195678710938, + "KL/std": 90.48197174072266, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.006202667951583862, + "fcm_dpo/delta": -0.018561676144599915, + "fcm_dpo/margin": 67.3564682006836, + "fcm_dpo/q_t": 0.40901321172714233, + "grad_norm": 22.224632263183594, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.36728447675704956, + "logits/rejected": -0.3497485816478729, + "logps/chosen": -140.07711791992188, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -227.83016967773438, + "loss": 1.1139, + "margin_dpo/margin_mean": 67.35647583007812, + "margin_dpo/margin_std": 109.0289306640625, + "step": 163 + }, + { + "KL/chosen_KL_mean": -78.06917572021484, + "KL/mean": -119.80747985839844, + "KL/rejected_KL_mean": -161.5457763671875, + "KL/std": 89.12982177734375, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.006050167605280876, + "fcm_dpo/delta": -0.11239530146121979, + "fcm_dpo/margin": 83.47660064697266, + "fcm_dpo/q_t": 0.38624992966651917, + "grad_norm": 22.583763122558594, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.36094290018081665, + "logits/rejected": -0.34708279371261597, + "logps/chosen": -133.7098388671875, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -241.21041870117188, + "loss": 1.031, + "margin_dpo/margin_mean": 83.47660064697266, + "margin_dpo/margin_std": 102.37945556640625, + "step": 164 + }, + { + "KL/chosen_KL_mean": -83.80238342285156, + "KL/mean": -110.8072280883789, + "KL/rejected_KL_mean": -137.81207275390625, + "KL/std": 75.36154174804688, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.006107107736170292, + "fcm_dpo/delta": 0.07260017096996307, + "fcm_dpo/margin": 54.009674072265625, + "fcm_dpo/q_t": 0.4261144995689392, + "grad_norm": 23.670085906982422, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.3795357942581177, + "logits/rejected": -0.35248273611068726, + "logps/chosen": -145.11306762695312, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -211.482666015625, + "loss": 1.1707, + "margin_dpo/margin_mean": 54.009674072265625, + "margin_dpo/margin_std": 99.5179443359375, + "step": 165 + }, + { + "KL/chosen_KL_mean": -73.61637115478516, + "KL/mean": -123.27507019042969, + "KL/rejected_KL_mean": -172.93377685546875, + "KL/std": 108.5833740234375, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.005936107598245144, + "fcm_dpo/delta": -0.20280691981315613, + "fcm_dpo/margin": 99.31739807128906, + "fcm_dpo/q_t": 0.3791484236717224, + "grad_norm": 17.751134872436523, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.3589398264884949, + "logits/rejected": -0.36944496631622314, + "logps/chosen": -124.5999755859375, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -271.0289001464844, + "loss": 1.0133, + "margin_dpo/margin_mean": 99.31739807128906, + "margin_dpo/margin_std": 136.53958129882812, + "step": 166 + }, + { + "KL/chosen_KL_mean": -74.20712280273438, + "KL/mean": -126.51481628417969, + "KL/rejected_KL_mean": -178.822509765625, + "KL/std": 99.15591430664062, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.005717212799936533, + "fcm_dpo/delta": -0.21081513166427612, + "fcm_dpo/margin": 104.61538696289062, + "fcm_dpo/q_t": 0.36653923988342285, + "grad_norm": 23.1616268157959, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.33332428336143494, + "logits/rejected": -0.33558595180511475, + "logps/chosen": -124.63121795654297, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -274.8529357910156, + "loss": 0.9691, + "margin_dpo/margin_mean": 104.61538696289062, + "margin_dpo/margin_std": 114.18727111816406, + "step": 167 + }, + { + "KL/chosen_KL_mean": -80.6678466796875, + "KL/mean": -119.04971313476562, + "KL/rejected_KL_mean": -157.43157958984375, + "KL/std": 92.95712280273438, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.005618114024400711, + "fcm_dpo/delta": -0.03284159675240517, + "fcm_dpo/margin": 76.76372528076172, + "fcm_dpo/q_t": 0.403435617685318, + "grad_norm": 21.089401245117188, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.3479578495025635, + "logits/rejected": -0.3380032181739807, + "logps/chosen": -130.13067626953125, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -232.74014282226562, + "loss": 1.0778, + "margin_dpo/margin_mean": 76.76372528076172, + "margin_dpo/margin_std": 104.18408203125, + "step": 168 + }, + { + "KL/chosen_KL_mean": -82.71928405761719, + "KL/mean": -116.30401611328125, + "KL/rejected_KL_mean": -149.88873291015625, + "KL/std": 91.65394592285156, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.005592959932982922, + "fcm_dpo/delta": 0.023978061974048615, + "fcm_dpo/margin": 67.16944885253906, + "fcm_dpo/q_t": 0.41969403624534607, + "grad_norm": 20.426271438598633, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.3647646903991699, + "logits/rejected": -0.34829095005989075, + "logps/chosen": -142.52273559570312, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -233.2344970703125, + "loss": 1.1594, + "margin_dpo/margin_mean": 67.16944885253906, + "margin_dpo/margin_std": 125.80655670166016, + "step": 169 + }, + { + "KL/chosen_KL_mean": -73.67047119140625, + "KL/mean": -113.36354064941406, + "KL/rejected_KL_mean": -153.0565948486328, + "KL/std": 87.04448699951172, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.005575606599450111, + "fcm_dpo/delta": -0.045632217079401016, + "fcm_dpo/margin": 79.38612365722656, + "fcm_dpo/q_t": 0.39885759353637695, + "grad_norm": 19.301816940307617, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.3231027126312256, + "logits/rejected": -0.3116713762283325, + "logps/chosen": -123.14225006103516, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -228.97393798828125, + "loss": 1.0726, + "margin_dpo/margin_mean": 79.38612365722656, + "margin_dpo/margin_std": 104.18345642089844, + "step": 170 + }, + { + "KL/chosen_KL_mean": -107.75534057617188, + "KL/mean": -139.4864501953125, + "KL/rejected_KL_mean": -171.21755981445312, + "KL/std": 101.39335632324219, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.005630516912788153, + "fcm_dpo/delta": 0.04427427425980568, + "fcm_dpo/margin": 63.46221160888672, + "fcm_dpo/q_t": 0.4247584044933319, + "grad_norm": 31.603731155395508, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.3770429491996765, + "logits/rejected": -0.3610066771507263, + "logps/chosen": -192.25465393066406, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -280.59967041015625, + "loss": 1.1771, + "margin_dpo/margin_mean": 63.462215423583984, + "margin_dpo/margin_std": 130.3548583984375, + "step": 171 + }, + { + "KL/chosen_KL_mean": -94.41339111328125, + "KL/mean": -127.90815734863281, + "KL/rejected_KL_mean": -161.40292358398438, + "KL/std": 97.80350494384766, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.005657501053065062, + "fcm_dpo/delta": 0.02182912454009056, + "fcm_dpo/margin": 66.98953247070312, + "fcm_dpo/q_t": 0.4152892827987671, + "grad_norm": 22.06650161743164, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.40264075994491577, + "logits/rejected": -0.3859758973121643, + "logps/chosen": -163.0673065185547, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -246.83958435058594, + "loss": 1.1316, + "margin_dpo/margin_mean": 66.98953247070312, + "margin_dpo/margin_std": 110.39387512207031, + "step": 172 + }, + { + "KL/chosen_KL_mean": -86.4061279296875, + "KL/mean": -120.02232360839844, + "KL/rejected_KL_mean": -153.6385498046875, + "KL/std": 91.43634033203125, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.00570608489215374, + "fcm_dpo/delta": 0.016680259257555008, + "fcm_dpo/margin": 67.23241424560547, + "fcm_dpo/q_t": 0.413169264793396, + "grad_norm": 20.329166412353516, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.4163011610507965, + "logits/rejected": -0.3886328935623169, + "logps/chosen": -149.45700073242188, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -232.32247924804688, + "loss": 1.1083, + "margin_dpo/margin_mean": 67.23241424560547, + "margin_dpo/margin_std": 96.06705474853516, + "step": 173 + }, + { + "KL/chosen_KL_mean": -82.22772979736328, + "KL/mean": -122.42765808105469, + "KL/rejected_KL_mean": -162.62759399414062, + "KL/std": 96.77005004882812, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.005674892105162144, + "fcm_dpo/delta": -0.05935989320278168, + "fcm_dpo/margin": 80.39986419677734, + "fcm_dpo/q_t": 0.4000336527824402, + "grad_norm": 22.312816619873047, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.33060014247894287, + "logits/rejected": -0.348196417093277, + "logps/chosen": -135.5906982421875, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -264.538818359375, + "loss": 1.0814, + "margin_dpo/margin_mean": 80.39985656738281, + "margin_dpo/margin_std": 115.71746063232422, + "step": 174 + }, + { + "KL/chosen_KL_mean": -75.79936218261719, + "KL/mean": -130.44143676757812, + "KL/rejected_KL_mean": -185.08352661132812, + "KL/std": 103.31723022460938, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.005444391630589962, + "fcm_dpo/delta": -0.20766407251358032, + "fcm_dpo/margin": 109.28416442871094, + "fcm_dpo/q_t": 0.3646450638771057, + "grad_norm": 27.097204208374023, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.3252009153366089, + "logits/rejected": -0.31189024448394775, + "logps/chosen": -121.21713256835938, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -274.58935546875, + "loss": 0.9536, + "margin_dpo/margin_mean": 109.28416442871094, + "margin_dpo/margin_std": 107.91986083984375, + "step": 175 + }, + { + "KL/chosen_KL_mean": -81.77299499511719, + "KL/mean": -127.77264404296875, + "KL/rejected_KL_mean": -173.77227783203125, + "KL/std": 102.29579162597656, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.00532842380926013, + "fcm_dpo/delta": -0.09475649893283844, + "fcm_dpo/margin": 91.99929809570312, + "fcm_dpo/q_t": 0.39243054389953613, + "grad_norm": 20.891366958618164, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.33610999584198, + "logits/rejected": -0.353465735912323, + "logps/chosen": -132.225830078125, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -269.33123779296875, + "loss": 1.0427, + "margin_dpo/margin_mean": 91.99929809570312, + "margin_dpo/margin_std": 118.4058837890625, + "step": 176 + }, + { + "KL/chosen_KL_mean": -95.29859924316406, + "KL/mean": -140.83084106445312, + "KL/rejected_KL_mean": -186.3630828857422, + "KL/std": 110.97512817382812, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.00524511095136404, + "fcm_dpo/delta": -0.08148036897182465, + "fcm_dpo/margin": 91.06448364257812, + "fcm_dpo/q_t": 0.3942405581474304, + "grad_norm": 26.494930267333984, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.3483707308769226, + "logits/rejected": -0.3397047519683838, + "logps/chosen": -156.5150604248047, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -282.2568664550781, + "loss": 1.0487, + "margin_dpo/margin_mean": 91.06449127197266, + "margin_dpo/margin_std": 118.02874755859375, + "step": 177 + }, + { + "KL/chosen_KL_mean": -105.19111633300781, + "KL/mean": -162.428466796875, + "KL/rejected_KL_mean": -219.6658172607422, + "KL/std": 130.05609130859375, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.005042451433837414, + "fcm_dpo/delta": -0.18919949233531952, + "fcm_dpo/margin": 114.47470092773438, + "fcm_dpo/q_t": 0.37493717670440674, + "grad_norm": 25.647525787353516, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.23960041999816895, + "logits/rejected": -0.23009046912193298, + "logps/chosen": -163.45590209960938, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -325.0311584472656, + "loss": 0.9989, + "margin_dpo/margin_mean": 114.47470092773438, + "margin_dpo/margin_std": 142.2547149658203, + "step": 178 + }, + { + "KL/chosen_KL_mean": -108.95854187011719, + "KL/mean": -149.76498413085938, + "KL/rejected_KL_mean": -190.5714111328125, + "KL/std": 112.00431823730469, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.004985122475773096, + "fcm_dpo/delta": -0.007611282169818878, + "fcm_dpo/margin": 81.61286926269531, + "fcm_dpo/q_t": 0.4105031490325928, + "grad_norm": 30.070934295654297, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.281255304813385, + "logits/rejected": -0.28930869698524475, + "logps/chosen": -170.01687622070312, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -281.0992431640625, + "loss": 1.132, + "margin_dpo/margin_mean": 81.61286926269531, + "margin_dpo/margin_std": 139.04403686523438, + "step": 179 + }, + { + "KL/chosen_KL_mean": -93.63102722167969, + "KL/mean": -146.12298583984375, + "KL/rejected_KL_mean": -198.61492919921875, + "KL/std": 100.87710571289062, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.004909930285066366, + "fcm_dpo/delta": -0.12193223834037781, + "fcm_dpo/margin": 104.98388671875, + "fcm_dpo/q_t": 0.3830623924732208, + "grad_norm": 20.473424911499023, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.2962295114994049, + "logits/rejected": -0.2902482748031616, + "logps/chosen": -147.9737548828125, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -296.8267822265625, + "loss": 1.021, + "margin_dpo/margin_mean": 104.98388671875, + "margin_dpo/margin_std": 125.305908203125, + "step": 180 + }, + { + "KL/chosen_KL_mean": -81.89930725097656, + "KL/mean": -114.54834747314453, + "KL/rejected_KL_mean": -147.1973876953125, + "KL/std": 92.09681701660156, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.0049502113834023476, + "fcm_dpo/delta": 0.07918453216552734, + "fcm_dpo/margin": 65.29808044433594, + "fcm_dpo/q_t": 0.4265759587287903, + "grad_norm": 17.70340919494629, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.3195858299732208, + "logits/rejected": -0.2943240702152252, + "logps/chosen": -136.89976501464844, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -208.85354614257812, + "loss": 1.1663, + "margin_dpo/margin_mean": 65.29808044433594, + "margin_dpo/margin_std": 116.14714050292969, + "step": 181 + }, + { + "KL/chosen_KL_mean": -78.28688049316406, + "KL/mean": -134.71717834472656, + "KL/rejected_KL_mean": -191.14747619628906, + "KL/std": 111.72691345214844, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.004830837715417147, + "fcm_dpo/delta": -0.1545790135860443, + "fcm_dpo/margin": 112.86061096191406, + "fcm_dpo/q_t": 0.37639355659484863, + "grad_norm": 18.506088256835938, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.28846609592437744, + "logits/rejected": -0.2981233596801758, + "logps/chosen": -119.39472961425781, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -280.6690673828125, + "loss": 1.0109, + "margin_dpo/margin_mean": 112.86061096191406, + "margin_dpo/margin_std": 134.60597229003906, + "step": 182 + }, + { + "KL/chosen_KL_mean": -114.15873718261719, + "KL/mean": -144.86376953125, + "KL/rejected_KL_mean": -175.56878662109375, + "KL/std": 92.10914611816406, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.004841846413910389, + "fcm_dpo/delta": 0.10491826385259628, + "fcm_dpo/margin": 61.41004943847656, + "fcm_dpo/q_t": 0.4317929148674011, + "grad_norm": 20.767650604248047, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.3042393922805786, + "logits/rejected": -0.2935281991958618, + "logps/chosen": -171.68328857421875, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -251.5445098876953, + "loss": 1.1784, + "margin_dpo/margin_mean": 61.410057067871094, + "margin_dpo/margin_std": 101.00328826904297, + "step": 183 + }, + { + "KL/chosen_KL_mean": -95.66822814941406, + "KL/mean": -127.72172546386719, + "KL/rejected_KL_mean": -159.7752227783203, + "KL/std": 86.71656036376953, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.005001405254006386, + "fcm_dpo/delta": 0.08096103370189667, + "fcm_dpo/margin": 64.10699462890625, + "fcm_dpo/q_t": 0.4279393255710602, + "grad_norm": 21.993793487548828, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.31411975622177124, + "logits/rejected": -0.3029560446739197, + "logps/chosen": -154.2131805419922, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -236.40927124023438, + "loss": 1.1629, + "margin_dpo/margin_mean": 64.10699462890625, + "margin_dpo/margin_std": 108.95437622070312, + "step": 184 + }, + { + "KL/chosen_KL_mean": -103.86957550048828, + "KL/mean": -129.41604614257812, + "KL/rejected_KL_mean": -154.9625244140625, + "KL/std": 100.53556823730469, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.005100230686366558, + "fcm_dpo/delta": 0.14321333169937134, + "fcm_dpo/margin": 51.09294891357422, + "fcm_dpo/q_t": 0.44449546933174133, + "grad_norm": 21.621137619018555, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.3124648928642273, + "logits/rejected": -0.28814172744750977, + "logps/chosen": -165.89541625976562, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -228.72512817382812, + "loss": 1.2293, + "margin_dpo/margin_mean": 51.09294891357422, + "margin_dpo/margin_std": 121.37908935546875, + "step": 185 + }, + { + "KL/chosen_KL_mean": -94.98322296142578, + "KL/mean": -143.42227172851562, + "KL/rejected_KL_mean": -191.86129760742188, + "KL/std": 101.50787353515625, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.005060682073235512, + "fcm_dpo/delta": -0.09509499371051788, + "fcm_dpo/margin": 96.87808990478516, + "fcm_dpo/q_t": 0.38929298520088196, + "grad_norm": 28.849084854125977, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.3631850779056549, + "logits/rejected": -0.34014010429382324, + "logps/chosen": -164.33670043945312, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -279.93377685546875, + "loss": 1.032, + "margin_dpo/margin_mean": 96.87809753417969, + "margin_dpo/margin_std": 115.64311218261719, + "step": 186 + }, + { + "KL/chosen_KL_mean": -89.17625427246094, + "KL/mean": -129.23342895507812, + "KL/rejected_KL_mean": -169.29058837890625, + "KL/std": 96.46709442138672, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.005060152150690556, + "fcm_dpo/delta": -0.005964066833257675, + "fcm_dpo/margin": 80.11433410644531, + "fcm_dpo/q_t": 0.4074205160140991, + "grad_norm": 24.486600875854492, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.29304659366607666, + "logits/rejected": -0.28550541400909424, + "logps/chosen": -141.93270874023438, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -251.2596893310547, + "loss": 1.089, + "margin_dpo/margin_mean": 80.11431884765625, + "margin_dpo/margin_std": 105.45170593261719, + "step": 187 + }, + { + "KL/chosen_KL_mean": -82.52178955078125, + "KL/mean": -131.14796447753906, + "KL/rejected_KL_mean": -179.774169921875, + "KL/std": 107.0216064453125, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.004975754767656326, + "fcm_dpo/delta": -0.08811045438051224, + "fcm_dpo/margin": 97.25238037109375, + "fcm_dpo/q_t": 0.3912336230278015, + "grad_norm": 31.620702743530273, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.3411184847354889, + "logits/rejected": -0.34751665592193604, + "logps/chosen": -131.93727111816406, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -269.3145751953125, + "loss": 1.036, + "margin_dpo/margin_mean": 97.25238800048828, + "margin_dpo/margin_std": 117.13838958740234, + "step": 188 + }, + { + "KL/chosen_KL_mean": -95.95454406738281, + "KL/mean": -135.883056640625, + "KL/rejected_KL_mean": -175.81158447265625, + "KL/std": 106.31744384765625, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.0049576228484511375, + "fcm_dpo/delta": 0.004238678142428398, + "fcm_dpo/margin": 79.85704040527344, + "fcm_dpo/q_t": 0.4131699502468109, + "grad_norm": 31.445402145385742, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.34107866883277893, + "logits/rejected": -0.3217487335205078, + "logps/chosen": -148.35350036621094, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -247.9789276123047, + "loss": 1.1197, + "margin_dpo/margin_mean": 79.85704040527344, + "margin_dpo/margin_std": 130.01014709472656, + "step": 189 + }, + { + "KL/chosen_KL_mean": -102.51383972167969, + "KL/mean": -149.80548095703125, + "KL/rejected_KL_mean": -197.09707641601562, + "KL/std": 114.36456298828125, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.004943820182234049, + "fcm_dpo/delta": -0.07184645533561707, + "fcm_dpo/margin": 94.583251953125, + "fcm_dpo/q_t": 0.39662712812423706, + "grad_norm": 18.938213348388672, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.3630479574203491, + "logits/rejected": -0.3506428003311157, + "logps/chosen": -167.1968994140625, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -299.6476135253906, + "loss": 1.0763, + "margin_dpo/margin_mean": 94.583251953125, + "margin_dpo/margin_std": 132.47256469726562, + "step": 190 + }, + { + "KL/chosen_KL_mean": -94.4525146484375, + "KL/mean": -162.17788696289062, + "KL/rejected_KL_mean": -229.90325927734375, + "KL/std": 132.2121124267578, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.004684216342866421, + "fcm_dpo/delta": -0.2517712414264679, + "fcm_dpo/margin": 135.45077514648438, + "fcm_dpo/q_t": 0.36012277007102966, + "grad_norm": 21.33709716796875, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.3431745767593384, + "logits/rejected": -0.32149481773376465, + "logps/chosen": -163.11138916015625, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -340.0429382324219, + "loss": 0.9469, + "margin_dpo/margin_mean": 135.45077514648438, + "margin_dpo/margin_std": 145.202880859375, + "step": 191 + }, + { + "KL/chosen_KL_mean": -123.40989685058594, + "KL/mean": -168.1400146484375, + "KL/rejected_KL_mean": -212.8701171875, + "KL/std": 115.60923767089844, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.004643384367227554, + "fcm_dpo/delta": -0.01615481823682785, + "fcm_dpo/margin": 89.46023559570312, + "fcm_dpo/q_t": 0.408674031496048, + "grad_norm": 25.144752502441406, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.32124069333076477, + "logits/rejected": -0.3204975128173828, + "logps/chosen": -193.13681030273438, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -316.19146728515625, + "loss": 1.1119, + "margin_dpo/margin_mean": 89.46023559570312, + "margin_dpo/margin_std": 142.5316925048828, + "step": 192 + }, + { + "KL/chosen_KL_mean": -124.36747741699219, + "KL/mean": -150.41734313964844, + "KL/rejected_KL_mean": -176.46719360351562, + "KL/std": 106.24888610839844, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.004757573828101158, + "fcm_dpo/delta": 0.15488943457603455, + "fcm_dpo/margin": 52.099708557128906, + "fcm_dpo/q_t": 0.4419249892234802, + "grad_norm": 26.762622833251953, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.322111040353775, + "logits/rejected": -0.31205689907073975, + "logps/chosen": -184.5579833984375, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -252.874755859375, + "loss": 1.2527, + "margin_dpo/margin_mean": 52.099708557128906, + "margin_dpo/margin_std": 135.6827850341797, + "step": 193 + }, + { + "KL/chosen_KL_mean": -77.10400390625, + "KL/mean": -120.11808776855469, + "KL/rejected_KL_mean": -163.1321563720703, + "KL/std": 88.75360107421875, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.004753452725708485, + "fcm_dpo/delta": -0.009496469050645828, + "fcm_dpo/margin": 86.02815246582031, + "fcm_dpo/q_t": 0.40442323684692383, + "grad_norm": 19.890329360961914, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.22950759530067444, + "logits/rejected": -0.21171870827674866, + "logps/chosen": -114.94438171386719, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -223.8169403076172, + "loss": 1.0695, + "margin_dpo/margin_mean": 86.02815246582031, + "margin_dpo/margin_std": 97.06770324707031, + "step": 194 + }, + { + "KL/chosen_KL_mean": -122.94843292236328, + "KL/mean": -168.73760986328125, + "KL/rejected_KL_mean": -214.52676391601562, + "KL/std": 110.55722045898438, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.004738887771964073, + "fcm_dpo/delta": -0.03552384674549103, + "fcm_dpo/margin": 91.57832336425781, + "fcm_dpo/q_t": 0.4007643461227417, + "grad_norm": 22.172483444213867, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.2802814841270447, + "logits/rejected": -0.29949259757995605, + "logps/chosen": -177.83999633789062, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -311.2977294921875, + "loss": 1.066, + "margin_dpo/margin_mean": 91.57832336425781, + "margin_dpo/margin_std": 114.41627502441406, + "step": 195 + }, + { + "KL/chosen_KL_mean": -96.11858367919922, + "KL/mean": -148.2082061767578, + "KL/rejected_KL_mean": -200.29782104492188, + "KL/std": 112.41001892089844, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.0046433815732598305, + "fcm_dpo/delta": -0.08907097578048706, + "fcm_dpo/margin": 104.17922973632812, + "fcm_dpo/q_t": 0.39215317368507385, + "grad_norm": 18.21831703186035, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.33319342136383057, + "logits/rejected": -0.31911373138427734, + "logps/chosen": -149.36383056640625, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -276.35076904296875, + "loss": 1.0531, + "margin_dpo/margin_mean": 104.17922973632812, + "margin_dpo/margin_std": 134.5558319091797, + "step": 196 + }, + { + "KL/chosen_KL_mean": -98.41443634033203, + "KL/mean": -138.86544799804688, + "KL/rejected_KL_mean": -179.31646728515625, + "KL/std": 97.470458984375, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.004677818156778812, + "fcm_dpo/delta": 0.022110365331172943, + "fcm_dpo/margin": 80.90202331542969, + "fcm_dpo/q_t": 0.413760781288147, + "grad_norm": 19.29098129272461, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.3001037836074829, + "logits/rejected": -0.2938286364078522, + "logps/chosen": -158.83477783203125, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -256.5253601074219, + "loss": 1.106, + "margin_dpo/margin_mean": 80.90202331542969, + "margin_dpo/margin_std": 110.55290222167969, + "step": 197 + }, + { + "KL/chosen_KL_mean": -110.12991333007812, + "KL/mean": -161.87913513183594, + "KL/rejected_KL_mean": -213.62835693359375, + "KL/std": 122.3536376953125, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.004613672383129597, + "fcm_dpo/delta": -0.08136218786239624, + "fcm_dpo/margin": 103.4984130859375, + "fcm_dpo/q_t": 0.3950771689414978, + "grad_norm": 20.996286392211914, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.31790387630462646, + "logits/rejected": -0.31772375106811523, + "logps/chosen": -165.16610717773438, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -310.8716125488281, + "loss": 1.0538, + "margin_dpo/margin_mean": 103.4984130859375, + "margin_dpo/margin_std": 139.29356384277344, + "step": 198 + }, + { + "KL/chosen_KL_mean": -105.38700866699219, + "KL/mean": -157.11480712890625, + "KL/rejected_KL_mean": -208.84259033203125, + "KL/std": 111.04532623291016, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.004517151974141598, + "fcm_dpo/delta": -0.0717441737651825, + "fcm_dpo/margin": 103.45556640625, + "fcm_dpo/q_t": 0.394577294588089, + "grad_norm": 27.031187057495117, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.3286276161670685, + "logits/rejected": -0.3115686774253845, + "logps/chosen": -162.21583557128906, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -293.49078369140625, + "loss": 1.0588, + "margin_dpo/margin_mean": 103.45557403564453, + "margin_dpo/margin_std": 134.02767944335938, + "step": 199 + }, + { + "KL/chosen_KL_mean": -103.82003784179688, + "KL/mean": -155.0573272705078, + "KL/rejected_KL_mean": -206.2946014404297, + "KL/std": 119.82543182373047, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.004469861276447773, + "fcm_dpo/delta": -0.061313264071941376, + "fcm_dpo/margin": 102.47457885742188, + "fcm_dpo/q_t": 0.3992011845111847, + "grad_norm": 22.639179229736328, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.2985066771507263, + "logits/rejected": -0.2816792130470276, + "logps/chosen": -156.8870849609375, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -286.9030456542969, + "loss": 1.084, + "margin_dpo/margin_mean": 102.47457885742188, + "margin_dpo/margin_std": 152.56202697753906, + "step": 200 + }, + { + "KL/chosen_KL_mean": -108.69739532470703, + "KL/mean": -159.94659423828125, + "KL/rejected_KL_mean": -211.19580078125, + "KL/std": 125.89083862304688, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.0044439565390348434, + "fcm_dpo/delta": -0.05812525749206543, + "fcm_dpo/margin": 102.4984130859375, + "fcm_dpo/q_t": 0.3977164030075073, + "grad_norm": 20.81543731689453, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.337613582611084, + "logits/rejected": -0.3325209617614746, + "logps/chosen": -184.099609375, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -326.0040283203125, + "loss": 1.0722, + "margin_dpo/margin_mean": 102.4984130859375, + "margin_dpo/margin_std": 141.6723175048828, + "step": 201 + }, + { + "KL/chosen_KL_mean": -111.74082946777344, + "KL/mean": -147.52963256835938, + "KL/rejected_KL_mean": -183.3184051513672, + "KL/std": 108.16849517822266, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.004478315822780132, + "fcm_dpo/delta": 0.08200031518936157, + "fcm_dpo/margin": 71.57759094238281, + "fcm_dpo/q_t": 0.4288518726825714, + "grad_norm": 21.68619728088379, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.30491214990615845, + "logits/rejected": -0.3188881278038025, + "logps/chosen": -161.84214782714844, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -270.3034362792969, + "loss": 1.1818, + "margin_dpo/margin_mean": 71.57759094238281, + "margin_dpo/margin_std": 139.62728881835938, + "step": 202 + }, + { + "KL/chosen_KL_mean": -111.330078125, + "KL/mean": -155.8389892578125, + "KL/rejected_KL_mean": -200.347900390625, + "KL/std": 111.86740112304688, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.0044879126362502575, + "fcm_dpo/delta": 0.0004851408302783966, + "fcm_dpo/margin": 89.01782989501953, + "fcm_dpo/q_t": 0.40893062949180603, + "grad_norm": 22.087295532226562, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.33827900886535645, + "logits/rejected": -0.32895606756210327, + "logps/chosen": -171.9397735595703, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -286.243896484375, + "loss": 1.0954, + "margin_dpo/margin_mean": 89.01783752441406, + "margin_dpo/margin_std": 122.31471252441406, + "step": 203 + }, + { + "KL/chosen_KL_mean": -123.50933837890625, + "KL/mean": -164.68695068359375, + "KL/rejected_KL_mean": -205.86456298828125, + "KL/std": 121.135009765625, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.004510943777859211, + "fcm_dpo/delta": 0.02961009368300438, + "fcm_dpo/margin": 82.355224609375, + "fcm_dpo/q_t": 0.41832876205444336, + "grad_norm": 23.451541900634766, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.4115005135536194, + "logits/rejected": -0.3721199035644531, + "logps/chosen": -203.67430114746094, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -293.56048583984375, + "loss": 1.1416, + "margin_dpo/margin_mean": 82.355224609375, + "margin_dpo/margin_std": 140.3298797607422, + "step": 204 + }, + { + "KL/chosen_KL_mean": -118.86104583740234, + "KL/mean": -173.5120391845703, + "KL/rejected_KL_mean": -228.16302490234375, + "KL/std": 124.20469665527344, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.004422344267368317, + "fcm_dpo/delta": -0.08987303078174591, + "fcm_dpo/margin": 109.3019790649414, + "fcm_dpo/q_t": 0.38937753438949585, + "grad_norm": 21.78606605529785, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.33596229553222656, + "logits/rejected": -0.30631476640701294, + "logps/chosen": -178.24578857421875, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -313.2880859375, + "loss": 1.0494, + "margin_dpo/margin_mean": 109.30197143554688, + "margin_dpo/margin_std": 135.16091918945312, + "step": 205 + }, + { + "KL/chosen_KL_mean": -108.62220764160156, + "KL/mean": -167.1552276611328, + "KL/rejected_KL_mean": -225.68826293945312, + "KL/std": 116.17403411865234, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.004349352326244116, + "fcm_dpo/delta": -0.11564525961875916, + "fcm_dpo/margin": 117.06604766845703, + "fcm_dpo/q_t": 0.3817872405052185, + "grad_norm": 25.174524307250977, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.26543641090393066, + "logits/rejected": -0.26824256777763367, + "logps/chosen": -155.58670043945312, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -324.6417541503906, + "loss": 1.011, + "margin_dpo/margin_mean": 117.06604766845703, + "margin_dpo/margin_std": 126.22386169433594, + "step": 206 + }, + { + "KL/chosen_KL_mean": -99.20771026611328, + "KL/mean": -166.33590698242188, + "KL/rejected_KL_mean": -233.46409606933594, + "KL/std": 132.5963592529297, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.004230023827403784, + "fcm_dpo/delta": -0.17784729599952698, + "fcm_dpo/margin": 134.25637817382812, + "fcm_dpo/q_t": 0.37461385130882263, + "grad_norm": 23.600038528442383, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.3762626647949219, + "logits/rejected": -0.3513372540473938, + "logps/chosen": -155.26397705078125, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -317.9118957519531, + "loss": 0.989, + "margin_dpo/margin_mean": 134.25637817382812, + "margin_dpo/margin_std": 154.54873657226562, + "step": 207 + }, + { + "KL/chosen_KL_mean": -153.62962341308594, + "KL/mean": -206.26727294921875, + "KL/rejected_KL_mean": -258.9049377441406, + "KL/std": 126.89285278320312, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.0041542090475559235, + "fcm_dpo/delta": -0.039197564125061035, + "fcm_dpo/margin": 105.27531433105469, + "fcm_dpo/q_t": 0.40104472637176514, + "grad_norm": 25.210905075073242, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.3824605345726013, + "logits/rejected": -0.3646228015422821, + "logps/chosen": -220.69723510742188, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -353.19183349609375, + "loss": 1.0902, + "margin_dpo/margin_mean": 105.27531433105469, + "margin_dpo/margin_std": 153.97900390625, + "step": 208 + }, + { + "KL/chosen_KL_mean": -129.7876434326172, + "KL/mean": -176.7648468017578, + "KL/rejected_KL_mean": -223.74203491210938, + "KL/std": 115.69462585449219, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.004161862190812826, + "fcm_dpo/delta": 0.009330113418400288, + "fcm_dpo/margin": 93.95440673828125, + "fcm_dpo/q_t": 0.4116600453853607, + "grad_norm": 31.72983169555664, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.25245344638824463, + "logits/rejected": -0.2391929030418396, + "logps/chosen": -185.96932983398438, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -304.6835632324219, + "loss": 1.124, + "margin_dpo/margin_mean": 93.95440673828125, + "margin_dpo/margin_std": 148.0377960205078, + "step": 209 + }, + { + "KL/chosen_KL_mean": -116.9127426147461, + "KL/mean": -170.77340698242188, + "KL/rejected_KL_mean": -224.63406372070312, + "KL/std": 118.03720092773438, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.0041490281000733376, + "fcm_dpo/delta": -0.049295347183942795, + "fcm_dpo/margin": 107.72132110595703, + "fcm_dpo/q_t": 0.39966607093811035, + "grad_norm": 24.604143142700195, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.29855215549468994, + "logits/rejected": -0.29052990674972534, + "logps/chosen": -163.28456115722656, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -301.315673828125, + "loss": 1.0718, + "margin_dpo/margin_mean": 107.72132110595703, + "margin_dpo/margin_std": 144.06710815429688, + "step": 210 + }, + { + "KL/chosen_KL_mean": -167.2972412109375, + "KL/mean": -213.8275146484375, + "KL/rejected_KL_mean": -260.3577880859375, + "KL/std": 135.11312866210938, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.004132451023906469, + "fcm_dpo/delta": 0.016039201989769936, + "fcm_dpo/margin": 93.06055450439453, + "fcm_dpo/q_t": 0.4170151352882385, + "grad_norm": 33.378421783447266, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.311035692691803, + "logits/rejected": -0.2705526351928711, + "logps/chosen": -246.22958374023438, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -347.17877197265625, + "loss": 1.1383, + "margin_dpo/margin_mean": 93.06055450439453, + "margin_dpo/margin_std": 158.61228942871094, + "step": 211 + }, + { + "KL/chosen_KL_mean": -137.9550018310547, + "KL/mean": -204.14096069335938, + "KL/rejected_KL_mean": -270.3269348144531, + "KL/std": 145.64797973632812, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.004017878323793411, + "fcm_dpo/delta": -0.14147897064685822, + "fcm_dpo/margin": 132.37193298339844, + "fcm_dpo/q_t": 0.38083723187446594, + "grad_norm": 27.12974739074707, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.3167477250099182, + "logits/rejected": -0.3083157539367676, + "logps/chosen": -196.15200805664062, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -373.384765625, + "loss": 1.0263, + "margin_dpo/margin_mean": 132.37193298339844, + "margin_dpo/margin_std": 160.85980224609375, + "step": 212 + }, + { + "KL/chosen_KL_mean": -131.25747680664062, + "KL/mean": -193.85498046875, + "KL/rejected_KL_mean": -256.4524841308594, + "KL/std": 127.0328369140625, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.003945375792682171, + "fcm_dpo/delta": -0.09945961833000183, + "fcm_dpo/margin": 125.19500732421875, + "fcm_dpo/q_t": 0.3875296711921692, + "grad_norm": 32.48273468017578, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.3081282675266266, + "logits/rejected": -0.2790905833244324, + "logps/chosen": -198.77020263671875, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -350.3671875, + "loss": 1.0344, + "margin_dpo/margin_mean": 125.19501495361328, + "margin_dpo/margin_std": 150.89865112304688, + "step": 213 + }, + { + "KL/chosen_KL_mean": -114.31307220458984, + "KL/mean": -170.82357788085938, + "KL/rejected_KL_mean": -227.33407592773438, + "KL/std": 125.99368286132812, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.003913573920726776, + "fcm_dpo/delta": -0.04427690804004669, + "fcm_dpo/margin": 113.02099609375, + "fcm_dpo/q_t": 0.4000810980796814, + "grad_norm": 23.648649215698242, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.2536253333091736, + "logits/rejected": -0.2575327455997467, + "logps/chosen": -155.91796875, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -304.85150146484375, + "loss": 1.0674, + "margin_dpo/margin_mean": 113.02099609375, + "margin_dpo/margin_std": 146.56906127929688, + "step": 214 + }, + { + "KL/chosen_KL_mean": -132.95579528808594, + "KL/mean": -191.78021240234375, + "KL/rejected_KL_mean": -250.60464477539062, + "KL/std": 128.5931854248047, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.0038622168358415365, + "fcm_dpo/delta": -0.05724211037158966, + "fcm_dpo/margin": 117.64883422851562, + "fcm_dpo/q_t": 0.39529117941856384, + "grad_norm": 25.225217819213867, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.28956931829452515, + "logits/rejected": -0.28073662519454956, + "logps/chosen": -186.2350616455078, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -340.56927490234375, + "loss": 1.0452, + "margin_dpo/margin_mean": 117.64884185791016, + "margin_dpo/margin_std": 133.9748992919922, + "step": 215 + }, + { + "KL/chosen_KL_mean": -137.43809509277344, + "KL/mean": -196.4094696044922, + "KL/rejected_KL_mean": -255.380859375, + "KL/std": 135.23500061035156, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.0038448460400104523, + "fcm_dpo/delta": -0.05614541471004486, + "fcm_dpo/margin": 117.94276428222656, + "fcm_dpo/q_t": 0.3985063433647156, + "grad_norm": 23.17816162109375, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.28292590379714966, + "logits/rejected": -0.2649373412132263, + "logps/chosen": -186.32589721679688, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -332.57977294921875, + "loss": 1.0782, + "margin_dpo/margin_mean": 117.94276428222656, + "margin_dpo/margin_std": 165.9866943359375, + "step": 216 + }, + { + "KL/chosen_KL_mean": -138.44813537597656, + "KL/mean": -207.92813110351562, + "KL/rejected_KL_mean": -277.40814208984375, + "KL/std": 137.0869140625, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.003740387037396431, + "fcm_dpo/delta": -0.12657949328422546, + "fcm_dpo/margin": 138.95999145507812, + "fcm_dpo/q_t": 0.3821297585964203, + "grad_norm": 21.533512115478516, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.301749587059021, + "logits/rejected": -0.29722312092781067, + "logps/chosen": -188.29344177246094, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -377.4864501953125, + "loss": 1.0061, + "margin_dpo/margin_mean": 138.95999145507812, + "margin_dpo/margin_std": 153.20114135742188, + "step": 217 + }, + { + "KL/chosen_KL_mean": -146.38714599609375, + "KL/mean": -200.0693359375, + "KL/rejected_KL_mean": -253.75152587890625, + "KL/std": 137.21426391601562, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.0037157037295401096, + "fcm_dpo/delta": 0.001042960211634636, + "fcm_dpo/margin": 107.3644027709961, + "fcm_dpo/q_t": 0.41018977761268616, + "grad_norm": 21.820951461791992, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.30885905027389526, + "logits/rejected": -0.30377811193466187, + "logps/chosen": -204.9638214111328, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -341.5979309082031, + "loss": 1.1076, + "margin_dpo/margin_mean": 107.3644027709961, + "margin_dpo/margin_std": 158.10263061523438, + "step": 218 + }, + { + "KL/chosen_KL_mean": -157.92381286621094, + "KL/mean": -205.0916748046875, + "KL/rejected_KL_mean": -252.259521484375, + "KL/std": 144.50390625, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.0037665944546461105, + "fcm_dpo/delta": 0.045606352388858795, + "fcm_dpo/margin": 94.335693359375, + "fcm_dpo/q_t": 0.42079412937164307, + "grad_norm": 30.14975929260254, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.3331577181816101, + "logits/rejected": -0.32641366124153137, + "logps/chosen": -219.00767517089844, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -338.0899658203125, + "loss": 1.1641, + "margin_dpo/margin_mean": 94.335693359375, + "margin_dpo/margin_std": 175.06253051757812, + "step": 219 + }, + { + "KL/chosen_KL_mean": -177.99212646484375, + "KL/mean": -216.95718383789062, + "KL/rejected_KL_mean": -255.92222595214844, + "KL/std": 123.7259521484375, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.0038101901300251484, + "fcm_dpo/delta": 0.10635118186473846, + "fcm_dpo/margin": 77.93009185791016, + "fcm_dpo/q_t": 0.4304437041282654, + "grad_norm": 29.36302947998047, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.3234555125236511, + "logits/rejected": -0.3008239269256592, + "logps/chosen": -248.02342224121094, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -343.60772705078125, + "loss": 1.1674, + "margin_dpo/margin_mean": 77.93008422851562, + "margin_dpo/margin_std": 126.6876449584961, + "step": 220 + }, + { + "KL/chosen_KL_mean": -152.09149169921875, + "KL/mean": -237.8167724609375, + "KL/rejected_KL_mean": -323.5420227050781, + "KL/std": 157.6136474609375, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.0037078431341797113, + "fcm_dpo/delta": -0.25117895007133484, + "fcm_dpo/margin": 171.45053100585938, + "fcm_dpo/q_t": 0.3553469777107239, + "grad_norm": 28.02921485900879, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.27162307500839233, + "logits/rejected": -0.2771974205970764, + "logps/chosen": -204.2462158203125, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -430.00970458984375, + "loss": 0.9311, + "margin_dpo/margin_mean": 171.45053100585938, + "margin_dpo/margin_std": 161.2157440185547, + "step": 221 + }, + { + "KL/chosen_KL_mean": -159.20440673828125, + "KL/mean": -223.759033203125, + "KL/rejected_KL_mean": -288.31365966796875, + "KL/std": 145.17880249023438, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.0036267125979065895, + "fcm_dpo/delta": -0.07167011499404907, + "fcm_dpo/margin": 129.10926818847656, + "fcm_dpo/q_t": 0.39351508021354675, + "grad_norm": 20.42731285095215, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.3312653601169586, + "logits/rejected": -0.332950234413147, + "logps/chosen": -220.17550659179688, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -388.3148193359375, + "loss": 1.0511, + "margin_dpo/margin_mean": 129.10926818847656, + "margin_dpo/margin_std": 160.65625, + "step": 222 + }, + { + "KL/chosen_KL_mean": -170.2442626953125, + "KL/mean": -224.3243408203125, + "KL/rejected_KL_mean": -278.4044189453125, + "KL/std": 143.28465270996094, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.003599923336878419, + "fcm_dpo/delta": 0.0110526904463768, + "fcm_dpo/margin": 108.16015625, + "fcm_dpo/q_t": 0.4126368463039398, + "grad_norm": 25.651758193969727, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.22283935546875, + "logits/rejected": -0.21641430258750916, + "logps/chosen": -222.8848419189453, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -361.22943115234375, + "loss": 1.1325, + "margin_dpo/margin_mean": 108.16015625, + "margin_dpo/margin_std": 181.91226196289062, + "step": 223 + }, + { + "KL/chosen_KL_mean": -147.8184814453125, + "KL/mean": -209.83712768554688, + "KL/rejected_KL_mean": -271.8558349609375, + "KL/std": 157.56280517578125, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.003555042203515768, + "fcm_dpo/delta": -0.044342927634716034, + "fcm_dpo/margin": 124.03733825683594, + "fcm_dpo/q_t": 0.4030151665210724, + "grad_norm": 27.533445358276367, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.2727223038673401, + "logits/rejected": -0.25196322798728943, + "logps/chosen": -196.41387939453125, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -348.9722900390625, + "loss": 1.0896, + "margin_dpo/margin_mean": 124.03733825683594, + "margin_dpo/margin_std": 181.13235473632812, + "step": 224 + }, + { + "KL/chosen_KL_mean": -168.59347534179688, + "KL/mean": -242.18316650390625, + "KL/rejected_KL_mean": -315.7728576660156, + "KL/std": 153.1339111328125, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.0035054893232882023, + "fcm_dpo/delta": -0.12240298092365265, + "fcm_dpo/margin": 147.17938232421875, + "fcm_dpo/q_t": 0.3844594657421112, + "grad_norm": 24.890352249145508, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.30197474360466003, + "logits/rejected": -0.2900247573852539, + "logps/chosen": -226.59393310546875, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -415.67578125, + "loss": 1.0308, + "margin_dpo/margin_mean": 147.17938232421875, + "margin_dpo/margin_std": 186.26852416992188, + "step": 225 + }, + { + "KL/chosen_KL_mean": -142.2608642578125, + "KL/mean": -202.4642333984375, + "KL/rejected_KL_mean": -262.6676025390625, + "KL/std": 151.34628295898438, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.0034564360976219177, + "fcm_dpo/delta": -0.01779451221227646, + "fcm_dpo/margin": 120.40673828125, + "fcm_dpo/q_t": 0.4076775908470154, + "grad_norm": 31.96993637084961, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.36338162422180176, + "logits/rejected": -0.3226342797279358, + "logps/chosen": -201.15966796875, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -341.3553466796875, + "loss": 1.098, + "margin_dpo/margin_mean": 120.40673828125, + "margin_dpo/margin_std": 174.12762451171875, + "step": 226 + }, + { + "KL/chosen_KL_mean": -165.43106079101562, + "KL/mean": -238.38694763183594, + "KL/rejected_KL_mean": -311.34283447265625, + "KL/std": 167.44435119628906, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.0034084515646100044, + "fcm_dpo/delta": -0.10316304862499237, + "fcm_dpo/margin": 145.91177368164062, + "fcm_dpo/q_t": 0.3868604302406311, + "grad_norm": 23.062564849853516, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.3930385112762451, + "logits/rejected": -0.3825671374797821, + "logps/chosen": -224.50323486328125, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -410.75518798828125, + "loss": 1.029, + "margin_dpo/margin_mean": 145.91177368164062, + "margin_dpo/margin_std": 173.82473754882812, + "step": 227 + }, + { + "KL/chosen_KL_mean": -165.41331481933594, + "KL/mean": -217.64407348632812, + "KL/rejected_KL_mean": -269.87481689453125, + "KL/std": 136.05606079101562, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.003430598881095648, + "fcm_dpo/delta": 0.043085500597953796, + "fcm_dpo/margin": 104.46148681640625, + "fcm_dpo/q_t": 0.41795143485069275, + "grad_norm": 24.486732482910156, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.3330225944519043, + "logits/rejected": -0.31356292963027954, + "logps/chosen": -231.30459594726562, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -360.923583984375, + "loss": 1.1361, + "margin_dpo/margin_mean": 104.46148681640625, + "margin_dpo/margin_std": 166.13107299804688, + "step": 228 + }, + { + "KL/chosen_KL_mean": -165.3111572265625, + "KL/mean": -220.4169921875, + "KL/rejected_KL_mean": -275.5228576660156, + "KL/std": 153.3724822998047, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.0034456737339496613, + "fcm_dpo/delta": 0.021033309400081635, + "fcm_dpo/margin": 110.21168518066406, + "fcm_dpo/q_t": 0.41243353486061096, + "grad_norm": 31.090543746948242, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.43453967571258545, + "logits/rejected": -0.40375208854675293, + "logps/chosen": -236.01751708984375, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -360.0502624511719, + "loss": 1.1139, + "margin_dpo/margin_mean": 110.21168518066406, + "margin_dpo/margin_std": 161.30140686035156, + "step": 229 + }, + { + "KL/chosen_KL_mean": -128.0115966796875, + "KL/mean": -210.7613525390625, + "KL/rejected_KL_mean": -293.5111083984375, + "KL/std": 146.32284545898438, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.003359460271894932, + "fcm_dpo/delta": -0.16545212268829346, + "fcm_dpo/margin": 165.49949645996094, + "fcm_dpo/q_t": 0.37107786536216736, + "grad_norm": 28.731571197509766, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.3344762325286865, + "logits/rejected": -0.3386141061782837, + "logps/chosen": -167.29360961914062, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -379.13299560546875, + "loss": 0.9722, + "margin_dpo/margin_mean": 165.49948120117188, + "margin_dpo/margin_std": 160.73019409179688, + "step": 230 + }, + { + "KL/chosen_KL_mean": -147.03927612304688, + "KL/mean": -204.8795166015625, + "KL/rejected_KL_mean": -262.7197265625, + "KL/std": 130.2517852783203, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.0033422503620386124, + "fcm_dpo/delta": 0.01390581950545311, + "fcm_dpo/margin": 115.68045806884766, + "fcm_dpo/q_t": 0.41191643476486206, + "grad_norm": 24.250959396362305, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.39889034628868103, + "logits/rejected": -0.36870649456977844, + "logps/chosen": -210.31573486328125, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -336.8436279296875, + "loss": 1.0946, + "margin_dpo/margin_mean": 115.68045806884766, + "margin_dpo/margin_std": 150.3603057861328, + "step": 231 + }, + { + "KL/chosen_KL_mean": -182.9542236328125, + "KL/mean": -229.5906524658203, + "KL/rejected_KL_mean": -276.22705078125, + "KL/std": 152.603271484375, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.003382818540558219, + "fcm_dpo/delta": 0.08731842041015625, + "fcm_dpo/margin": 93.27284240722656, + "fcm_dpo/q_t": 0.42943495512008667, + "grad_norm": 25.97873306274414, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.40977489948272705, + "logits/rejected": -0.3872949481010437, + "logps/chosen": -253.70298767089844, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -360.2041320800781, + "loss": 1.1598, + "margin_dpo/margin_mean": 93.27284240722656, + "margin_dpo/margin_std": 156.10711669921875, + "step": 232 + }, + { + "KL/chosen_KL_mean": -163.78579711914062, + "KL/mean": -236.087890625, + "KL/rejected_KL_mean": -308.3899841308594, + "KL/std": 160.25900268554688, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.003360921749845147, + "fcm_dpo/delta": -0.09034262597560883, + "fcm_dpo/margin": 144.60418701171875, + "fcm_dpo/q_t": 0.3940209150314331, + "grad_norm": 24.915483474731445, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.39276862144470215, + "logits/rejected": -0.39590269327163696, + "logps/chosen": -218.66873168945312, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -415.87005615234375, + "loss": 1.0644, + "margin_dpo/margin_mean": 144.60418701171875, + "margin_dpo/margin_std": 205.10748291015625, + "step": 233 + }, + { + "KL/chosen_KL_mean": -159.46267700195312, + "KL/mean": -233.42210388183594, + "KL/rejected_KL_mean": -307.38153076171875, + "KL/std": 140.7521514892578, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.003276141593232751, + "fcm_dpo/delta": -0.09045238792896271, + "fcm_dpo/margin": 147.91888427734375, + "fcm_dpo/q_t": 0.38880056142807007, + "grad_norm": 45.84584045410156, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.3655707836151123, + "logits/rejected": -0.3900142312049866, + "logps/chosen": -203.55718994140625, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -407.38818359375, + "loss": 1.0252, + "margin_dpo/margin_mean": 147.91888427734375, + "margin_dpo/margin_std": 159.43338012695312, + "step": 234 + }, + { + "KL/chosen_KL_mean": -191.55911254882812, + "KL/mean": -240.37464904785156, + "KL/rejected_KL_mean": -289.190185546875, + "KL/std": 135.47872924804688, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0033297771587967873, + "fcm_dpo/delta": 0.07708244025707245, + "fcm_dpo/margin": 97.63107299804688, + "fcm_dpo/q_t": 0.42509645223617554, + "grad_norm": 24.456079483032227, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.4143291413784027, + "logits/rejected": -0.39424288272857666, + "logps/chosen": -253.79702758789062, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -379.5852355957031, + "loss": 1.1561, + "margin_dpo/margin_mean": 97.63107299804688, + "margin_dpo/margin_std": 160.899658203125, + "step": 235 + }, + { + "KL/chosen_KL_mean": -141.00631713867188, + "KL/mean": -218.29336547851562, + "KL/rejected_KL_mean": -295.58038330078125, + "KL/std": 146.28057861328125, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.003279463853687048, + "fcm_dpo/delta": -0.11260546743869781, + "fcm_dpo/margin": 154.57408142089844, + "fcm_dpo/q_t": 0.3809179663658142, + "grad_norm": 46.03508377075195, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.39001888036727905, + "logits/rejected": -0.3897605538368225, + "logps/chosen": -190.34768676757812, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -399.092041015625, + "loss": 0.9899, + "margin_dpo/margin_mean": 154.57408142089844, + "margin_dpo/margin_std": 137.58041381835938, + "step": 236 + }, + { + "KL/chosen_KL_mean": -186.0025634765625, + "KL/mean": -246.36611938476562, + "KL/rejected_KL_mean": -306.7297058105469, + "KL/std": 149.6922607421875, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0032669170759618282, + "fcm_dpo/delta": 0.00578081700950861, + "fcm_dpo/margin": 120.72715759277344, + "fcm_dpo/q_t": 0.41017407178878784, + "grad_norm": 26.383832931518555, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.4057920575141907, + "logits/rejected": -0.39955034852027893, + "logps/chosen": -240.17068481445312, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -401.51007080078125, + "loss": 1.1037, + "margin_dpo/margin_mean": 120.72715759277344, + "margin_dpo/margin_std": 173.07215881347656, + "step": 237 + }, + { + "KL/chosen_KL_mean": -165.48025512695312, + "KL/mean": -225.99395751953125, + "KL/rejected_KL_mean": -286.5076904296875, + "KL/std": 149.29281616210938, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0032509847078472376, + "fcm_dpo/delta": 0.006365515291690826, + "fcm_dpo/margin": 121.02743530273438, + "fcm_dpo/q_t": 0.4122035801410675, + "grad_norm": 23.1090030670166, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.36196550726890564, + "logits/rejected": -0.3773908317089081, + "logps/chosen": -219.453369140625, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -375.9256286621094, + "loss": 1.1004, + "margin_dpo/margin_mean": 121.02743530273438, + "margin_dpo/margin_std": 168.41207885742188, + "step": 238 + }, + { + "KL/chosen_KL_mean": -177.00595092773438, + "KL/mean": -243.75650024414062, + "KL/rejected_KL_mean": -310.50701904296875, + "KL/std": 139.99700927734375, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.0032305652275681496, + "fcm_dpo/delta": -0.0337536484003067, + "fcm_dpo/margin": 133.50108337402344, + "fcm_dpo/q_t": 0.40004390478134155, + "grad_norm": 40.00619888305664, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.3681572377681732, + "logits/rejected": -0.3547469973564148, + "logps/chosen": -235.103759765625, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -404.0999755859375, + "loss": 1.0659, + "margin_dpo/margin_mean": 133.50108337402344, + "margin_dpo/margin_std": 157.92364501953125, + "step": 239 + }, + { + "KL/chosen_KL_mean": -190.62051391601562, + "KL/mean": -247.27947998046875, + "KL/rejected_KL_mean": -303.9384460449219, + "KL/std": 154.82113647460938, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.003252096241340041, + "fcm_dpo/delta": 0.032605692744255066, + "fcm_dpo/margin": 113.31793212890625, + "fcm_dpo/q_t": 0.4164040684700012, + "grad_norm": 39.15624237060547, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.39388927817344666, + "logits/rejected": -0.3720082640647888, + "logps/chosen": -251.23501586914062, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -378.0569763183594, + "loss": 1.1379, + "margin_dpo/margin_mean": 113.31791687011719, + "margin_dpo/margin_std": 185.2668914794922, + "step": 240 + }, + { + "KL/chosen_KL_mean": -161.00314331054688, + "KL/mean": -249.38790893554688, + "KL/rejected_KL_mean": -337.7726745605469, + "KL/std": 171.18887329101562, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0031904843635857105, + "fcm_dpo/delta": -0.17351412773132324, + "fcm_dpo/margin": 176.76953125, + "fcm_dpo/q_t": 0.3746398091316223, + "grad_norm": 27.225372314453125, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.40906286239624023, + "logits/rejected": -0.38160303235054016, + "logps/chosen": -227.09420776367188, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -425.83355712890625, + "loss": 0.9912, + "margin_dpo/margin_mean": 176.76953125, + "margin_dpo/margin_std": 198.92465209960938, + "step": 241 + }, + { + "KL/chosen_KL_mean": -186.9355926513672, + "KL/mean": -249.46463012695312, + "KL/rejected_KL_mean": -311.99365234375, + "KL/std": 146.27365112304688, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.0031714560464024544, + "fcm_dpo/delta": 0.0030847005546092987, + "fcm_dpo/margin": 125.0580825805664, + "fcm_dpo/q_t": 0.4105117917060852, + "grad_norm": 35.89213943481445, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.36399373412132263, + "logits/rejected": -0.3402002453804016, + "logps/chosen": -254.79953002929688, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -395.35400390625, + "loss": 1.0981, + "margin_dpo/margin_mean": 125.05806732177734, + "margin_dpo/margin_std": 172.0503387451172, + "step": 242 + }, + { + "KL/chosen_KL_mean": -193.0344696044922, + "KL/mean": -270.7990417480469, + "KL/rejected_KL_mean": -348.5636291503906, + "KL/std": 168.34158325195312, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.0031119785271584988, + "fcm_dpo/delta": -0.08843523263931274, + "fcm_dpo/margin": 155.52914428710938, + "fcm_dpo/q_t": 0.38993075489997864, + "grad_norm": 22.183713912963867, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.36588960886001587, + "logits/rejected": -0.33492511510849, + "logps/chosen": -256.11871337890625, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -424.8992614746094, + "loss": 1.0366, + "margin_dpo/margin_mean": 155.52914428710938, + "margin_dpo/margin_std": 188.0260009765625, + "step": 243 + }, + { + "KL/chosen_KL_mean": -169.53494262695312, + "KL/mean": -249.1429443359375, + "KL/rejected_KL_mean": -328.7509765625, + "KL/std": 159.25384521484375, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.003054366447031498, + "fcm_dpo/delta": -0.09097754955291748, + "fcm_dpo/margin": 159.21604919433594, + "fcm_dpo/q_t": 0.3877517580986023, + "grad_norm": 35.43682861328125, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.4336305856704712, + "logits/rejected": -0.4257158041000366, + "logps/chosen": -230.67562866210938, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -423.6429138183594, + "loss": 1.0168, + "margin_dpo/margin_mean": 159.21604919433594, + "margin_dpo/margin_std": 164.11810302734375, + "step": 244 + }, + { + "KL/chosen_KL_mean": -189.56396484375, + "KL/mean": -252.20779418945312, + "KL/rejected_KL_mean": -314.85162353515625, + "KL/std": 153.77020263671875, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0030314174946397543, + "fcm_dpo/delta": 0.01989227533340454, + "fcm_dpo/margin": 125.28765869140625, + "fcm_dpo/q_t": 0.4126083254814148, + "grad_norm": 27.907432556152344, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.40659791231155396, + "logits/rejected": -0.37769126892089844, + "logps/chosen": -256.82623291015625, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -402.49169921875, + "loss": 1.1099, + "margin_dpo/margin_mean": 125.28765869140625, + "margin_dpo/margin_std": 169.4322052001953, + "step": 245 + }, + { + "KL/chosen_KL_mean": -181.76121520996094, + "KL/mean": -245.38681030273438, + "KL/rejected_KL_mean": -309.01239013671875, + "KL/std": 168.49429321289062, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.0030555007979273796, + "fcm_dpo/delta": 0.01152346096932888, + "fcm_dpo/margin": 127.25117492675781, + "fcm_dpo/q_t": 0.4122322201728821, + "grad_norm": 29.177263259887695, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.4459696114063263, + "logits/rejected": -0.40770888328552246, + "logps/chosen": -248.45819091796875, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -393.3587341308594, + "loss": 1.1034, + "margin_dpo/margin_mean": 127.25117492675781, + "margin_dpo/margin_std": 179.67160034179688, + "step": 246 + }, + { + "KL/chosen_KL_mean": -211.78652954101562, + "KL/mean": -298.9569091796875, + "KL/rejected_KL_mean": -386.1273193359375, + "KL/std": 175.59890747070312, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.003002369310706854, + "fcm_dpo/delta": -0.1304749846458435, + "fcm_dpo/margin": 174.3407745361328, + "fcm_dpo/q_t": 0.3798842132091522, + "grad_norm": 30.964874267578125, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.38244399428367615, + "logits/rejected": -0.38297730684280396, + "logps/chosen": -268.3918762207031, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -492.42059326171875, + "loss": 1.005, + "margin_dpo/margin_mean": 174.3407745361328, + "margin_dpo/margin_std": 189.28936767578125, + "step": 247 + }, + { + "KL/chosen_KL_mean": -182.0874481201172, + "KL/mean": -260.6456298828125, + "KL/rejected_KL_mean": -339.2038269042969, + "KL/std": 145.6084442138672, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.0029460065998136997, + "fcm_dpo/delta": -0.0664525181055069, + "fcm_dpo/margin": 157.11636352539062, + "fcm_dpo/q_t": 0.39027613401412964, + "grad_norm": 23.116975784301758, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.3582613468170166, + "logits/rejected": -0.37669503688812256, + "logps/chosen": -226.1306610107422, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -431.0606994628906, + "loss": 1.0175, + "margin_dpo/margin_mean": 157.11636352539062, + "margin_dpo/margin_std": 140.55767822265625, + "step": 248 + }, + { + "KL/chosen_KL_mean": -233.33441162109375, + "KL/mean": -275.6315612792969, + "KL/rejected_KL_mean": -317.9286804199219, + "KL/std": 154.8394775390625, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0030127970967441797, + "fcm_dpo/delta": 0.14895941317081451, + "fcm_dpo/margin": 84.59425354003906, + "fcm_dpo/q_t": 0.44180774688720703, + "grad_norm": 34.42735290527344, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.4113328456878662, + "logits/rejected": -0.3859025239944458, + "logps/chosen": -295.7767639160156, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -398.396728515625, + "loss": 1.2177, + "margin_dpo/margin_mean": 84.59425354003906, + "margin_dpo/margin_std": 179.3761444091797, + "step": 249 + }, + { + "KL/chosen_KL_mean": -206.29498291015625, + "KL/mean": -285.16455078125, + "KL/rejected_KL_mean": -364.03411865234375, + "KL/std": 156.44363403320312, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.003012712113559246, + "fcm_dpo/delta": -0.07898565381765366, + "fcm_dpo/margin": 157.7391357421875, + "fcm_dpo/q_t": 0.3887900710105896, + "grad_norm": 36.26414108276367, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.38194775581359863, + "logits/rejected": -0.32314977049827576, + "logps/chosen": -271.93170166015625, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -437.90594482421875, + "loss": 1.0185, + "margin_dpo/margin_mean": 157.7391357421875, + "margin_dpo/margin_std": 154.3454132080078, + "step": 250 + }, + { + "KL/chosen_KL_mean": -219.74249267578125, + "KL/mean": -273.3251953125, + "KL/rejected_KL_mean": -326.90789794921875, + "KL/std": 164.439208984375, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.003016002243384719, + "fcm_dpo/delta": 0.07941900193691254, + "fcm_dpo/margin": 107.16543579101562, + "fcm_dpo/q_t": 0.42634153366088867, + "grad_norm": 26.837820053100586, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.3314884305000305, + "logits/rejected": -0.2961741089820862, + "logps/chosen": -276.9252014160156, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -404.57135009765625, + "loss": 1.1607, + "margin_dpo/margin_mean": 107.16543579101562, + "margin_dpo/margin_std": 177.86146545410156, + "step": 251 + }, + { + "KL/chosen_KL_mean": -219.5676727294922, + "KL/mean": -293.23724365234375, + "KL/rejected_KL_mean": -366.9068298339844, + "KL/std": 146.7100830078125, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.0030086305923759937, + "fcm_dpo/delta": -0.04552536457777023, + "fcm_dpo/margin": 147.33917236328125, + "fcm_dpo/q_t": 0.3941207528114319, + "grad_norm": 27.524452209472656, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.42891061305999756, + "logits/rejected": -0.38934314250946045, + "logps/chosen": -291.2533264160156, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -451.6648254394531, + "loss": 1.0388, + "margin_dpo/margin_mean": 147.3391571044922, + "margin_dpo/margin_std": 146.851318359375, + "step": 252 + }, + { + "KL/chosen_KL_mean": -183.92666625976562, + "KL/mean": -255.322509765625, + "KL/rejected_KL_mean": -326.7183532714844, + "KL/std": 154.73846435546875, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0030120660085231066, + "fcm_dpo/delta": -0.03195137903094292, + "fcm_dpo/margin": 142.79168701171875, + "fcm_dpo/q_t": 0.3995547890663147, + "grad_norm": 21.075923919677734, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.4324120879173279, + "logits/rejected": -0.4243052005767822, + "logps/chosen": -253.0605926513672, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -425.4208679199219, + "loss": 1.0684, + "margin_dpo/margin_mean": 142.79168701171875, + "margin_dpo/margin_std": 171.98837280273438, + "step": 253 + }, + { + "KL/chosen_KL_mean": -178.47354125976562, + "KL/mean": -239.57925415039062, + "KL/rejected_KL_mean": -300.6849670410156, + "KL/std": 161.2766876220703, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.003002177458256483, + "fcm_dpo/delta": 0.034374505281448364, + "fcm_dpo/margin": 122.21142578125, + "fcm_dpo/q_t": 0.4188271760940552, + "grad_norm": 21.827579498291016, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.39047229290008545, + "logits/rejected": -0.3732510209083557, + "logps/chosen": -232.6285400390625, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -390.99261474609375, + "loss": 1.1353, + "margin_dpo/margin_mean": 122.21144104003906, + "margin_dpo/margin_std": 201.87570190429688, + "step": 254 + }, + { + "KL/chosen_KL_mean": -178.7769775390625, + "KL/mean": -243.58656311035156, + "KL/rejected_KL_mean": -308.39617919921875, + "KL/std": 141.48419189453125, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.0030044354498386383, + "fcm_dpo/delta": 0.01077347807586193, + "fcm_dpo/margin": 129.6192169189453, + "fcm_dpo/q_t": 0.40978312492370605, + "grad_norm": 20.49843406677246, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.3835906982421875, + "logits/rejected": -0.372795045375824, + "logps/chosen": -235.91864013671875, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -398.604736328125, + "loss": 1.1034, + "margin_dpo/margin_mean": 129.61920166015625, + "margin_dpo/margin_std": 177.77389526367188, + "step": 255 + }, + { + "KL/chosen_KL_mean": -156.7086181640625, + "KL/mean": -226.1547393798828, + "KL/rejected_KL_mean": -295.60089111328125, + "KL/std": 152.43389892578125, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.0030087416525930166, + "fcm_dpo/delta": -0.018669530749320984, + "fcm_dpo/margin": 138.89224243164062, + "fcm_dpo/q_t": 0.4037402272224426, + "grad_norm": 26.31764793395996, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.3154336214065552, + "logits/rejected": -0.3220304846763611, + "logps/chosen": -211.87210083007812, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -388.163818359375, + "loss": 1.077, + "margin_dpo/margin_mean": 138.8922576904297, + "margin_dpo/margin_std": 176.8680419921875, + "step": 256 + }, + { + "KL/chosen_KL_mean": -156.92086791992188, + "KL/mean": -229.6694793701172, + "KL/rejected_KL_mean": -302.4180908203125, + "KL/std": 159.15985107421875, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.0029707145877182484, + "fcm_dpo/delta": -0.034777216613292694, + "fcm_dpo/margin": 145.4972381591797, + "fcm_dpo/q_t": 0.40124642848968506, + "grad_norm": 24.41628646850586, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.3648619055747986, + "logits/rejected": -0.35083144903182983, + "logps/chosen": -206.3445587158203, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -381.9560241699219, + "loss": 1.0712, + "margin_dpo/margin_mean": 145.4972381591797, + "margin_dpo/margin_std": 179.42453002929688, + "step": 257 + }, + { + "KL/chosen_KL_mean": -208.0568084716797, + "KL/mean": -279.328125, + "KL/rejected_KL_mean": -350.5994567871094, + "KL/std": 168.068115234375, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.002978472039103508, + "fcm_dpo/delta": -0.02566421590745449, + "fcm_dpo/margin": 142.5426483154297, + "fcm_dpo/q_t": 0.4016719460487366, + "grad_norm": 22.85698127746582, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.3376578688621521, + "logits/rejected": -0.33399754762649536, + "logps/chosen": -267.44091796875, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -446.5895690917969, + "loss": 1.0878, + "margin_dpo/margin_mean": 142.5426483154297, + "margin_dpo/margin_std": 198.8697509765625, + "step": 258 + }, + { + "KL/chosen_KL_mean": -201.46923828125, + "KL/mean": -261.9717712402344, + "KL/rejected_KL_mean": -322.4743347167969, + "KL/std": 161.67681884765625, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.0029780541080981493, + "fcm_dpo/delta": 0.04104078933596611, + "fcm_dpo/margin": 121.00508880615234, + "fcm_dpo/q_t": 0.41770946979522705, + "grad_norm": 24.31220054626465, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.4113248586654663, + "logits/rejected": -0.4077299237251282, + "logps/chosen": -254.29757690429688, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -411.6659851074219, + "loss": 1.1268, + "margin_dpo/margin_mean": 121.00508880615234, + "margin_dpo/margin_std": 180.1262969970703, + "step": 259 + }, + { + "KL/chosen_KL_mean": -208.21334838867188, + "KL/mean": -289.44940185546875, + "KL/rejected_KL_mean": -370.6854553222656, + "KL/std": 168.1861572265625, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.002967136912047863, + "fcm_dpo/delta": -0.08620242029428482, + "fcm_dpo/margin": 162.47210693359375, + "fcm_dpo/q_t": 0.39046740531921387, + "grad_norm": 30.800331115722656, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.36541640758514404, + "logits/rejected": -0.3810834288597107, + "logps/chosen": -255.63101196289062, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -465.7752380371094, + "loss": 1.0248, + "margin_dpo/margin_mean": 162.47210693359375, + "margin_dpo/margin_std": 178.96702575683594, + "step": 260 + }, + { + "KL/chosen_KL_mean": -217.9481201171875, + "KL/mean": -294.39154052734375, + "KL/rejected_KL_mean": -370.8349609375, + "KL/std": 183.4643096923828, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0029137469828128815, + "fcm_dpo/delta": -0.04788537323474884, + "fcm_dpo/margin": 152.8868408203125, + "fcm_dpo/q_t": 0.398873895406723, + "grad_norm": 21.25298309326172, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.30483314394950867, + "logits/rejected": -0.30327141284942627, + "logps/chosen": -270.9794921875, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -459.34991455078125, + "loss": 1.0697, + "margin_dpo/margin_mean": 152.8868408203125, + "margin_dpo/margin_std": 201.92605590820312, + "step": 261 + }, + { + "KL/chosen_KL_mean": -254.94554138183594, + "KL/mean": -318.1561584472656, + "KL/rejected_KL_mean": -381.36676025390625, + "KL/std": 169.34521484375, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.0029387224931269884, + "fcm_dpo/delta": 0.029167257249355316, + "fcm_dpo/margin": 126.42125701904297, + "fcm_dpo/q_t": 0.41399508714675903, + "grad_norm": 30.829885482788086, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.31988468766212463, + "logits/rejected": -0.2865649461746216, + "logps/chosen": -314.565673828125, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -467.7853088378906, + "loss": 1.1074, + "margin_dpo/margin_mean": 126.42124938964844, + "margin_dpo/margin_std": 169.61929321289062, + "step": 262 + }, + { + "KL/chosen_KL_mean": -232.35475158691406, + "KL/mean": -322.16357421875, + "KL/rejected_KL_mean": -411.972412109375, + "KL/std": 196.5563507080078, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.0028673994820564985, + "fcm_dpo/delta": -0.12211281061172485, + "fcm_dpo/margin": 179.61767578125, + "fcm_dpo/q_t": 0.3825952410697937, + "grad_norm": 28.401193618774414, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.3638797998428345, + "logits/rejected": -0.33898645639419556, + "logps/chosen": -291.77569580078125, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -508.8296203613281, + "loss": 1.0186, + "margin_dpo/margin_mean": 179.61767578125, + "margin_dpo/margin_std": 208.6827392578125, + "step": 263 + }, + { + "KL/chosen_KL_mean": -237.11749267578125, + "KL/mean": -313.3917236328125, + "KL/rejected_KL_mean": -389.666015625, + "KL/std": 179.01055908203125, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.002827351214364171, + "fcm_dpo/delta": -0.033898305147886276, + "fcm_dpo/margin": 152.54849243164062, + "fcm_dpo/q_t": 0.40215998888015747, + "grad_norm": 30.56130027770996, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.392653226852417, + "logits/rejected": -0.3807740807533264, + "logps/chosen": -299.8395690917969, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -483.522216796875, + "loss": 1.0798, + "margin_dpo/margin_mean": 152.54849243164062, + "margin_dpo/margin_std": 202.1197509765625, + "step": 264 + }, + { + "KL/chosen_KL_mean": -259.13104248046875, + "KL/mean": -331.8868408203125, + "KL/rejected_KL_mean": -404.6426696777344, + "KL/std": 202.21392822265625, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.002835554303601384, + "fcm_dpo/delta": -0.013152186758816242, + "fcm_dpo/margin": 145.5115966796875, + "fcm_dpo/q_t": 0.4084446132183075, + "grad_norm": 32.036468505859375, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.3428543508052826, + "logits/rejected": -0.32623937726020813, + "logps/chosen": -321.1025390625, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -492.66326904296875, + "loss": 1.1229, + "margin_dpo/margin_mean": 145.5115966796875, + "margin_dpo/margin_std": 243.45236206054688, + "step": 265 + }, + { + "KL/chosen_KL_mean": -254.78836059570312, + "KL/mean": -313.7867736816406, + "KL/rejected_KL_mean": -372.78515625, + "KL/std": 166.00534057617188, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0028631887398660183, + "fcm_dpo/delta": 0.06428281962871552, + "fcm_dpo/margin": 117.99678039550781, + "fcm_dpo/q_t": 0.42279738187789917, + "grad_norm": 57.453773498535156, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.41433748602867126, + "logits/rejected": -0.37577420473098755, + "logps/chosen": -321.8880310058594, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -440.7563781738281, + "loss": 1.1461, + "margin_dpo/margin_mean": 117.99678039550781, + "margin_dpo/margin_std": 190.17715454101562, + "step": 266 + }, + { + "KL/chosen_KL_mean": -226.574462890625, + "KL/mean": -299.79608154296875, + "KL/rejected_KL_mean": -373.01763916015625, + "KL/std": 184.03443908691406, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.002853271085768938, + "fcm_dpo/delta": -0.018912963569164276, + "fcm_dpo/margin": 146.44320678710938, + "fcm_dpo/q_t": 0.4045720100402832, + "grad_norm": 28.506731033325195, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.4168621003627777, + "logits/rejected": -0.3850215673446655, + "logps/chosen": -295.54522705078125, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -463.18609619140625, + "loss": 1.0934, + "margin_dpo/margin_mean": 146.44320678710938, + "margin_dpo/margin_std": 208.85545349121094, + "step": 267 + }, + { + "KL/chosen_KL_mean": -226.53416442871094, + "KL/mean": -297.07427978515625, + "KL/rejected_KL_mean": -367.6143798828125, + "KL/std": 168.32553100585938, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0028516328893601894, + "fcm_dpo/delta": -0.002548152580857277, + "fcm_dpo/margin": 141.0802001953125, + "fcm_dpo/q_t": 0.408683717250824, + "grad_norm": 29.27951431274414, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.37580791115760803, + "logits/rejected": -0.38187897205352783, + "logps/chosen": -282.4344787597656, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -469.26202392578125, + "loss": 1.0991, + "margin_dpo/margin_mean": 141.0802001953125, + "margin_dpo/margin_std": 201.3629150390625, + "step": 268 + }, + { + "KL/chosen_KL_mean": -240.99200439453125, + "KL/mean": -325.84503173828125, + "KL/rejected_KL_mean": -410.697998046875, + "KL/std": 170.60096740722656, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.0028264063876122236, + "fcm_dpo/delta": -0.08360470831394196, + "fcm_dpo/margin": 169.70602416992188, + "fcm_dpo/q_t": 0.39013510942459106, + "grad_norm": 30.065053939819336, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.41282427310943604, + "logits/rejected": -0.3927035331726074, + "logps/chosen": -311.03155517578125, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -518.04736328125, + "loss": 1.0485, + "margin_dpo/margin_mean": 169.70602416992188, + "margin_dpo/margin_std": 214.57313537597656, + "step": 269 + }, + { + "KL/chosen_KL_mean": -217.55038452148438, + "KL/mean": -280.44146728515625, + "KL/rejected_KL_mean": -343.33251953125, + "KL/std": 153.36929321289062, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.0028382984455674887, + "fcm_dpo/delta": 0.04427041485905647, + "fcm_dpo/margin": 125.78216552734375, + "fcm_dpo/q_t": 0.41805094480514526, + "grad_norm": 29.420982360839844, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.44115063548088074, + "logits/rejected": -0.4337437152862549, + "logps/chosen": -287.0838623046875, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -453.26116943359375, + "loss": 1.1332, + "margin_dpo/margin_mean": 125.78216552734375, + "margin_dpo/margin_std": 194.60586547851562, + "step": 270 + }, + { + "KL/chosen_KL_mean": -202.66600036621094, + "KL/mean": -281.4805908203125, + "KL/rejected_KL_mean": -360.295166015625, + "KL/std": 156.70175170898438, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.0028096665628254414, + "fcm_dpo/delta": -0.04514153301715851, + "fcm_dpo/margin": 157.62916564941406, + "fcm_dpo/q_t": 0.3977411985397339, + "grad_norm": 25.10634422302246, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.39153826236724854, + "logits/rejected": -0.3737758994102478, + "logps/chosen": -259.4305725097656, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -452.80902099609375, + "loss": 1.0486, + "margin_dpo/margin_mean": 157.62916564941406, + "margin_dpo/margin_std": 176.103759765625, + "step": 271 + }, + { + "KL/chosen_KL_mean": -189.6742401123047, + "KL/mean": -288.6821594238281, + "KL/rejected_KL_mean": -387.6900939941406, + "KL/std": 179.32241821289062, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.002758004702627659, + "fcm_dpo/delta": -0.15428122878074646, + "fcm_dpo/margin": 198.01583862304688, + "fcm_dpo/q_t": 0.3730362355709076, + "grad_norm": 38.49697494506836, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.32239818572998047, + "logits/rejected": -0.33270448446273804, + "logps/chosen": -239.17138671875, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -493.2328796386719, + "loss": 0.9769, + "margin_dpo/margin_mean": 198.01585388183594, + "margin_dpo/margin_std": 187.6432647705078, + "step": 272 + }, + { + "KL/chosen_KL_mean": -229.2264404296875, + "KL/mean": -327.6177978515625, + "KL/rejected_KL_mean": -426.0091552734375, + "KL/std": 186.09432983398438, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.0026639692950993776, + "fcm_dpo/delta": -0.13172578811645508, + "fcm_dpo/margin": 196.78271484375, + "fcm_dpo/q_t": 0.37926948070526123, + "grad_norm": 27.060304641723633, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.3462122976779938, + "logits/rejected": -0.33053290843963623, + "logps/chosen": -292.20184326171875, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -518.5077514648438, + "loss": 0.9995, + "margin_dpo/margin_mean": 196.78271484375, + "margin_dpo/margin_std": 207.626953125, + "step": 273 + }, + { + "KL/chosen_KL_mean": -280.7240295410156, + "KL/mean": -355.0863342285156, + "KL/rejected_KL_mean": -429.4486083984375, + "KL/std": 173.10992431640625, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.002663701307028532, + "fcm_dpo/delta": 0.0038175582885742188, + "fcm_dpo/margin": 148.72459411621094, + "fcm_dpo/q_t": 0.4090237021446228, + "grad_norm": 26.748149871826172, + "learning_rate": 3.75e-07, + "logits/chosen": -0.30344340205192566, + "logits/rejected": -0.2852343022823334, + "logps/chosen": -336.3917236328125, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -506.78167724609375, + "loss": 1.0943, + "margin_dpo/margin_mean": 148.72459411621094, + "margin_dpo/margin_std": 197.82537841796875, + "step": 274 + }, + { + "KL/chosen_KL_mean": -217.09286499023438, + "KL/mean": -295.29132080078125, + "KL/rejected_KL_mean": -373.48980712890625, + "KL/std": 174.23095703125, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.0026571936905384064, + "fcm_dpo/delta": -0.016356676816940308, + "fcm_dpo/margin": 156.3969268798828, + "fcm_dpo/q_t": 0.4039890468120575, + "grad_norm": 29.809680938720703, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.3512224555015564, + "logits/rejected": -0.3564579486846924, + "logps/chosen": -265.68756103515625, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -466.79351806640625, + "loss": 1.0781, + "margin_dpo/margin_mean": 156.39691162109375, + "margin_dpo/margin_std": 196.810302734375, + "step": 275 + }, + { + "KL/chosen_KL_mean": -240.04769897460938, + "KL/mean": -320.7686767578125, + "KL/rejected_KL_mean": -401.4896240234375, + "KL/std": 183.21725463867188, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.0026315764989703894, + "fcm_dpo/delta": -0.02611299231648445, + "fcm_dpo/margin": 161.4419403076172, + "fcm_dpo/q_t": 0.40360918641090393, + "grad_norm": 24.737987518310547, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.31632789969444275, + "logits/rejected": -0.2927018702030182, + "logps/chosen": -296.6251220703125, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -471.85528564453125, + "loss": 1.0823, + "margin_dpo/margin_mean": 161.4419403076172, + "margin_dpo/margin_std": 218.93309020996094, + "step": 276 + }, + { + "KL/chosen_KL_mean": -263.6834716796875, + "KL/mean": -344.73370361328125, + "KL/rejected_KL_mean": -425.783935546875, + "KL/std": 181.76675415039062, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.002629021182656288, + "fcm_dpo/delta": -0.02736707217991352, + "fcm_dpo/margin": 162.1004638671875, + "fcm_dpo/q_t": 0.4022486209869385, + "grad_norm": 28.87696647644043, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.33954834938049316, + "logits/rejected": -0.32351475954055786, + "logps/chosen": -319.9550476074219, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -518.6652221679688, + "loss": 1.081, + "margin_dpo/margin_mean": 162.1004638671875, + "margin_dpo/margin_std": 216.2222137451172, + "step": 277 + }, + { + "KL/chosen_KL_mean": -229.9246063232422, + "KL/mean": -323.7257385253906, + "KL/rejected_KL_mean": -417.52691650390625, + "KL/std": 189.4619140625, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.002578889951109886, + "fcm_dpo/delta": -0.0884208232164383, + "fcm_dpo/margin": 187.60232543945312, + "fcm_dpo/q_t": 0.38945531845092773, + "grad_norm": 30.89027214050293, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.33440476655960083, + "logits/rejected": -0.3410074710845947, + "logps/chosen": -282.8665466308594, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -508.7804870605469, + "loss": 1.0284, + "margin_dpo/margin_mean": 187.60231018066406, + "margin_dpo/margin_std": 212.30308532714844, + "step": 278 + }, + { + "KL/chosen_KL_mean": -261.62725830078125, + "KL/mean": -355.627197265625, + "KL/rejected_KL_mean": -449.62713623046875, + "KL/std": 200.82456970214844, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.0025280516128987074, + "fcm_dpo/delta": -0.08023825287818909, + "fcm_dpo/margin": 187.99981689453125, + "fcm_dpo/q_t": 0.39322513341903687, + "grad_norm": 28.196674346923828, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.2764171361923218, + "logits/rejected": -0.27654173970222473, + "logps/chosen": -310.2685852050781, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -537.4785766601562, + "loss": 1.0553, + "margin_dpo/margin_mean": 187.9998321533203, + "margin_dpo/margin_std": 245.06185913085938, + "step": 279 + }, + { + "KL/chosen_KL_mean": -250.97735595703125, + "KL/mean": -345.24200439453125, + "KL/rejected_KL_mean": -439.5067138671875, + "KL/std": 181.037841796875, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.002503362251445651, + "fcm_dpo/delta": -0.07555822283029556, + "fcm_dpo/margin": 188.52932739257812, + "fcm_dpo/q_t": 0.3895108103752136, + "grad_norm": 27.83134651184082, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.3044317960739136, + "logits/rejected": -0.30706459283828735, + "logps/chosen": -309.77447509765625, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -538.1255493164062, + "loss": 1.0269, + "margin_dpo/margin_mean": 188.52932739257812, + "margin_dpo/margin_std": 198.40838623046875, + "step": 280 + }, + { + "KL/chosen_KL_mean": -222.50982666015625, + "KL/mean": -307.035400390625, + "KL/rejected_KL_mean": -391.56097412109375, + "KL/std": 176.46922302246094, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0024947195779532194, + "fcm_dpo/delta": -0.022847846150398254, + "fcm_dpo/margin": 169.0511474609375, + "fcm_dpo/q_t": 0.39992254972457886, + "grad_norm": 22.2485408782959, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.35164666175842285, + "logits/rejected": -0.32886388897895813, + "logps/chosen": -277.99835205078125, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -472.44354248046875, + "loss": 1.0628, + "margin_dpo/margin_mean": 169.0511474609375, + "margin_dpo/margin_std": 189.1600341796875, + "step": 281 + }, + { + "KL/chosen_KL_mean": -249.17910766601562, + "KL/mean": -313.0265808105469, + "KL/rejected_KL_mean": -376.8740234375, + "KL/std": 177.6136474609375, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.0025219093076884747, + "fcm_dpo/delta": 0.08004093170166016, + "fcm_dpo/margin": 127.69493865966797, + "fcm_dpo/q_t": 0.42739179730415344, + "grad_norm": 25.303466796875, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.4209149479866028, + "logits/rejected": -0.40070730447769165, + "logps/chosen": -322.249267578125, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -472.2250061035156, + "loss": 1.1502, + "margin_dpo/margin_mean": 127.69493103027344, + "margin_dpo/margin_std": 204.48983764648438, + "step": 282 + }, + { + "KL/chosen_KL_mean": -252.92950439453125, + "KL/mean": -350.19256591796875, + "KL/rejected_KL_mean": -447.4556579589844, + "KL/std": 212.15618896484375, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.0024990858510136604, + "fcm_dpo/delta": -0.09048599004745483, + "fcm_dpo/margin": 194.52615356445312, + "fcm_dpo/q_t": 0.3864080011844635, + "grad_norm": 33.80406188964844, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.360754132270813, + "logits/rejected": -0.36511898040771484, + "logps/chosen": -314.82794189453125, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -544.4421997070312, + "loss": 1.0187, + "margin_dpo/margin_mean": 194.52615356445312, + "margin_dpo/margin_std": 200.5904541015625, + "step": 283 + }, + { + "KL/chosen_KL_mean": -246.3289794921875, + "KL/mean": -338.49151611328125, + "KL/rejected_KL_mean": -430.654052734375, + "KL/std": 200.68072509765625, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0024535893462598324, + "fcm_dpo/delta": -0.05490017682313919, + "fcm_dpo/margin": 184.3250732421875, + "fcm_dpo/q_t": 0.39539825916290283, + "grad_norm": 32.90118408203125, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.37419161200523376, + "logits/rejected": -0.3606506586074829, + "logps/chosen": -304.7645263671875, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -524.123291015625, + "loss": 1.0394, + "margin_dpo/margin_mean": 184.3250732421875, + "margin_dpo/margin_std": 200.16165161132812, + "step": 284 + }, + { + "KL/chosen_KL_mean": -282.43023681640625, + "KL/mean": -365.06097412109375, + "KL/rejected_KL_mean": -447.6917419433594, + "KL/std": 174.3252410888672, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.0024613402783870697, + "fcm_dpo/delta": -0.007577486336231232, + "fcm_dpo/margin": 165.2615203857422, + "fcm_dpo/q_t": 0.40381038188934326, + "grad_norm": 24.024824142456055, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.3554389476776123, + "logits/rejected": -0.3442623019218445, + "logps/chosen": -348.66241455078125, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -546.818603515625, + "loss": 1.0754, + "margin_dpo/margin_mean": 165.2615203857422, + "margin_dpo/margin_std": 190.17901611328125, + "step": 285 + }, + { + "KL/chosen_KL_mean": -291.14373779296875, + "KL/mean": -382.1997375488281, + "KL/rejected_KL_mean": -473.2557678222656, + "KL/std": 205.317626953125, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.002433116314932704, + "fcm_dpo/delta": -0.04507460445165634, + "fcm_dpo/margin": 182.1120147705078, + "fcm_dpo/q_t": 0.3971979022026062, + "grad_norm": 33.7503547668457, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.37160688638687134, + "logits/rejected": -0.3597394824028015, + "logps/chosen": -364.0947570800781, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -561.84423828125, + "loss": 1.0587, + "margin_dpo/margin_mean": 182.1120147705078, + "margin_dpo/margin_std": 219.98648071289062, + "step": 286 + }, + { + "KL/chosen_KL_mean": -266.22808837890625, + "KL/mean": -339.29608154296875, + "KL/rejected_KL_mean": -412.364013671875, + "KL/std": 175.1865234375, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.002445379737764597, + "fcm_dpo/delta": 0.044099729508161545, + "fcm_dpo/margin": 146.13592529296875, + "fcm_dpo/q_t": 0.4169973134994507, + "grad_norm": 29.56614112854004, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.3289335370063782, + "logits/rejected": -0.31285345554351807, + "logps/chosen": -327.7692565917969, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -490.0600891113281, + "loss": 1.1124, + "margin_dpo/margin_mean": 146.13592529296875, + "margin_dpo/margin_std": 192.2376251220703, + "step": 287 + }, + { + "KL/chosen_KL_mean": -263.48565673828125, + "KL/mean": -358.77593994140625, + "KL/rejected_KL_mean": -454.06622314453125, + "KL/std": 184.0616455078125, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.0024049854837358, + "fcm_dpo/delta": -0.06293891370296478, + "fcm_dpo/margin": 190.58058166503906, + "fcm_dpo/q_t": 0.3923218250274658, + "grad_norm": 24.630207061767578, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.35831576585769653, + "logits/rejected": -0.35033804178237915, + "logps/chosen": -320.1468811035156, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -541.4019165039062, + "loss": 1.0327, + "margin_dpo/margin_mean": 190.58059692382812, + "margin_dpo/margin_std": 187.75746154785156, + "step": 288 + }, + { + "KL/chosen_KL_mean": -254.29751586914062, + "KL/mean": -359.5520324707031, + "KL/rejected_KL_mean": -464.8065490722656, + "KL/std": 202.56654357910156, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.0023851064033806324, + "fcm_dpo/delta": -0.1073443591594696, + "fcm_dpo/margin": 210.50900268554688, + "fcm_dpo/q_t": 0.38456693291664124, + "grad_norm": 35.190921783447266, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.3048046827316284, + "logits/rejected": -0.30898788571357727, + "logps/chosen": -299.5279235839844, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -552.44921875, + "loss": 1.0123, + "margin_dpo/margin_mean": 210.50900268554688, + "margin_dpo/margin_std": 223.16616821289062, + "step": 289 + }, + { + "KL/chosen_KL_mean": -268.06353759765625, + "KL/mean": -374.0406494140625, + "KL/rejected_KL_mean": -480.01776123046875, + "KL/std": 215.24462890625, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.0023260314483195543, + "fcm_dpo/delta": -0.09812031686306, + "fcm_dpo/margin": 211.95425415039062, + "fcm_dpo/q_t": 0.38832902908325195, + "grad_norm": 22.924795150756836, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.3318750858306885, + "logits/rejected": -0.3504355847835541, + "logps/chosen": -323.5350341796875, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -596.726318359375, + "loss": 1.0307, + "margin_dpo/margin_mean": 211.95423889160156, + "margin_dpo/margin_std": 253.19345092773438, + "step": 290 + }, + { + "KL/chosen_KL_mean": -228.1102294921875, + "KL/mean": -321.82373046875, + "KL/rejected_KL_mean": -415.5372619628906, + "KL/std": 183.31234741210938, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.0023160562850534916, + "fcm_dpo/delta": -0.035824865102767944, + "fcm_dpo/margin": 187.42703247070312, + "fcm_dpo/q_t": 0.39872562885284424, + "grad_norm": 28.14525604248047, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3454960584640503, + "logits/rejected": -0.3366660475730896, + "logps/chosen": -291.37060546875, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -504.8343505859375, + "loss": 1.0618, + "margin_dpo/margin_mean": 187.42703247070312, + "margin_dpo/margin_std": 220.55035400390625, + "step": 291 + }, + { + "KL/chosen_KL_mean": -246.32920837402344, + "KL/mean": -348.30316162109375, + "KL/rejected_KL_mean": -450.27716064453125, + "KL/std": 219.12893676757812, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.0022866586223244667, + "fcm_dpo/delta": -0.06960747390985489, + "fcm_dpo/margin": 203.94796752929688, + "fcm_dpo/q_t": 0.3927977979183197, + "grad_norm": 21.986263275146484, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.3215014636516571, + "logits/rejected": -0.3085959255695343, + "logps/chosen": -300.24774169921875, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -540.238525390625, + "loss": 1.0521, + "margin_dpo/margin_mean": 203.94796752929688, + "margin_dpo/margin_std": 254.4894256591797, + "step": 292 + }, + { + "KL/chosen_KL_mean": -249.41552734375, + "KL/mean": -327.93878173828125, + "KL/rejected_KL_mean": -406.46209716796875, + "KL/std": 216.0584259033203, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.0022888637613505125, + "fcm_dpo/delta": 0.041920192539691925, + "fcm_dpo/margin": 157.04656982421875, + "fcm_dpo/q_t": 0.419266939163208, + "grad_norm": 58.68396759033203, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.34532225131988525, + "logits/rejected": -0.3398059010505676, + "logps/chosen": -309.79156494140625, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -484.3145446777344, + "loss": 1.1338, + "margin_dpo/margin_mean": 157.04656982421875, + "margin_dpo/margin_std": 250.26177978515625, + "step": 293 + }, + { + "KL/chosen_KL_mean": -229.08016967773438, + "KL/mean": -316.8381652832031, + "KL/rejected_KL_mean": -404.59613037109375, + "KL/std": 193.66787719726562, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.0022842655889689922, + "fcm_dpo/delta": -0.0010704807937145233, + "fcm_dpo/margin": 175.51597595214844, + "fcm_dpo/q_t": 0.40897324681282043, + "grad_norm": 26.596664428710938, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.33005979657173157, + "logits/rejected": -0.33502426743507385, + "logps/chosen": -277.167724609375, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -486.4931335449219, + "loss": 1.0946, + "margin_dpo/margin_mean": 175.51596069335938, + "margin_dpo/margin_std": 240.451416015625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -287.4351806640625, + "KL/mean": -386.6189270019531, + "KL/rejected_KL_mean": -485.80267333984375, + "KL/std": 224.32241821289062, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.002273797057569027, + "fcm_dpo/delta": -0.05342426896095276, + "fcm_dpo/margin": 198.36749267578125, + "fcm_dpo/q_t": 0.398009330034256, + "grad_norm": 38.03213882446289, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.3622977137565613, + "logits/rejected": -0.36646854877471924, + "logps/chosen": -337.35986328125, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -573.2589721679688, + "loss": 1.0679, + "margin_dpo/margin_mean": 198.36749267578125, + "margin_dpo/margin_std": 261.5985107421875, + "step": 295 + }, + { + "KL/chosen_KL_mean": -364.887939453125, + "KL/mean": -432.8410339355469, + "KL/rejected_KL_mean": -500.79412841796875, + "KL/std": 219.3538818359375, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.0022818311117589474, + "fcm_dpo/delta": 0.09251266717910767, + "fcm_dpo/margin": 135.90621948242188, + "fcm_dpo/q_t": 0.4296514093875885, + "grad_norm": 40.06826400756836, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.37161651253700256, + "logits/rejected": -0.3551409840583801, + "logps/chosen": -430.379150390625, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -593.8831787109375, + "loss": 1.1986, + "margin_dpo/margin_mean": 135.90621948242188, + "margin_dpo/margin_std": 278.2286376953125, + "step": 296 + }, + { + "KL/chosen_KL_mean": -324.11083984375, + "KL/mean": -404.9265441894531, + "KL/rejected_KL_mean": -485.7421875, + "KL/std": 208.7950439453125, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.0023123747669160366, + "fcm_dpo/delta": 0.027247175574302673, + "fcm_dpo/margin": 161.63131713867188, + "fcm_dpo/q_t": 0.41365888714790344, + "grad_norm": 41.08960723876953, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.3504769206047058, + "logits/rejected": -0.3559377193450928, + "logps/chosen": -380.58782958984375, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -580.8807373046875, + "loss": 1.1042, + "margin_dpo/margin_mean": 161.63133239746094, + "margin_dpo/margin_std": 212.53671264648438, + "step": 297 + }, + { + "KL/chosen_KL_mean": -344.7669677734375, + "KL/mean": -450.7606201171875, + "KL/rejected_KL_mean": -556.7542724609375, + "KL/std": 268.1529541015625, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.0022986368276178837, + "fcm_dpo/delta": -0.09174687415361404, + "fcm_dpo/margin": 211.98736572265625, + "fcm_dpo/q_t": 0.3944346606731415, + "grad_norm": 30.570646286010742, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.39513838291168213, + "logits/rejected": -0.40871572494506836, + "logps/chosen": -412.09210205078125, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -673.41650390625, + "loss": 1.0708, + "margin_dpo/margin_mean": 211.98736572265625, + "margin_dpo/margin_std": 314.4266357421875, + "step": 298 + }, + { + "KL/chosen_KL_mean": -282.16949462890625, + "KL/mean": -371.5263671875, + "KL/rejected_KL_mean": -460.88330078125, + "KL/std": 204.18020629882812, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.00228826398961246, + "fcm_dpo/delta": -0.01002436876296997, + "fcm_dpo/margin": 178.71377563476562, + "fcm_dpo/q_t": 0.4080577790737152, + "grad_norm": 41.88591766357422, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.3406634032726288, + "logits/rejected": -0.3468000292778015, + "logps/chosen": -331.131591796875, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -545.2115478515625, + "loss": 1.1011, + "margin_dpo/margin_mean": 178.71377563476562, + "margin_dpo/margin_std": 256.77471923828125, + "step": 299 + }, + { + "KL/chosen_KL_mean": -346.8251647949219, + "KL/mean": -456.17706298828125, + "KL/rejected_KL_mean": -565.5289306640625, + "KL/std": 241.0040740966797, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0022469365503638983, + "fcm_dpo/delta": -0.09602753818035126, + "fcm_dpo/margin": 218.7037811279297, + "fcm_dpo/q_t": 0.3885802626609802, + "grad_norm": 31.864181518554688, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.3921772241592407, + "logits/rejected": -0.3997608423233032, + "logps/chosen": -405.89886474609375, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -661.495361328125, + "loss": 1.0536, + "margin_dpo/margin_mean": 218.70379638671875, + "margin_dpo/margin_std": 289.6651611328125, + "step": 300 + }, + { + "KL/chosen_KL_mean": -284.9259033203125, + "KL/mean": -390.86212158203125, + "KL/rejected_KL_mean": -496.79840087890625, + "KL/std": 214.92518615722656, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.0022015944123268127, + "fcm_dpo/delta": -0.06998801231384277, + "fcm_dpo/margin": 211.87246704101562, + "fcm_dpo/q_t": 0.3944150507450104, + "grad_norm": 22.54869842529297, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.36549288034439087, + "logits/rejected": -0.3641670346260071, + "logps/chosen": -342.17529296875, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -589.1519775390625, + "loss": 1.0444, + "margin_dpo/margin_mean": 211.87246704101562, + "margin_dpo/margin_std": 255.36529541015625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -245.61285400390625, + "KL/mean": -332.1461486816406, + "KL/rejected_KL_mean": -418.6794128417969, + "KL/std": 188.73184204101562, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.0021967655047774315, + "fcm_dpo/delta": 0.02034899592399597, + "fcm_dpo/margin": 173.0665740966797, + "fcm_dpo/q_t": 0.4122311472892761, + "grad_norm": 22.378347396850586, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.3697757422924042, + "logits/rejected": -0.37573909759521484, + "logps/chosen": -296.81085205078125, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -515.90576171875, + "loss": 1.0987, + "margin_dpo/margin_mean": 173.0665740966797, + "margin_dpo/margin_std": 221.77536010742188, + "step": 302 + }, + { + "KL/chosen_KL_mean": -258.4994812011719, + "KL/mean": -342.93157958984375, + "KL/rejected_KL_mean": -427.3636169433594, + "KL/std": 210.8197784423828, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002211278770118952, + "fcm_dpo/delta": 0.027561839669942856, + "fcm_dpo/margin": 168.8641357421875, + "fcm_dpo/q_t": 0.41449251770973206, + "grad_norm": 22.01853370666504, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.4009031653404236, + "logits/rejected": -0.38807475566864014, + "logps/chosen": -325.21343994140625, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -514.30908203125, + "loss": 1.1181, + "margin_dpo/margin_mean": 168.86415100097656, + "margin_dpo/margin_std": 246.19589233398438, + "step": 303 + }, + { + "KL/chosen_KL_mean": -229.81298828125, + "KL/mean": -323.12774658203125, + "KL/rejected_KL_mean": -416.4424743652344, + "KL/std": 190.36843872070312, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.0022234409116208553, + "fcm_dpo/delta": -0.0158514566719532, + "fcm_dpo/margin": 186.6295166015625, + "fcm_dpo/q_t": 0.4009750485420227, + "grad_norm": 38.08213424682617, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.4314624071121216, + "logits/rejected": -0.4216075539588928, + "logps/chosen": -301.763671875, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -506.91448974609375, + "loss": 1.0491, + "margin_dpo/margin_mean": 186.6295166015625, + "margin_dpo/margin_std": 167.75634765625, + "step": 304 + }, + { + "KL/chosen_KL_mean": -240.20440673828125, + "KL/mean": -323.169677734375, + "KL/rejected_KL_mean": -406.1349182128906, + "KL/std": 208.11160278320312, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.0022129444405436516, + "fcm_dpo/delta": 0.03369864821434021, + "fcm_dpo/margin": 165.93048095703125, + "fcm_dpo/q_t": 0.4162192642688751, + "grad_norm": 26.336360931396484, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.4338701665401459, + "logits/rejected": -0.4168831706047058, + "logps/chosen": -306.9996337890625, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -498.8894958496094, + "loss": 1.1141, + "margin_dpo/margin_mean": 165.93048095703125, + "margin_dpo/margin_std": 227.7814483642578, + "step": 305 + }, + { + "KL/chosen_KL_mean": -232.7366180419922, + "KL/mean": -313.000244140625, + "KL/rejected_KL_mean": -393.263916015625, + "KL/std": 182.50186157226562, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.0022439081221818924, + "fcm_dpo/delta": 0.04121974855661392, + "fcm_dpo/margin": 160.5272979736328, + "fcm_dpo/q_t": 0.4157223105430603, + "grad_norm": 28.437631607055664, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.42782917618751526, + "logits/rejected": -0.4043479561805725, + "logps/chosen": -302.4205322265625, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -478.423095703125, + "loss": 1.1022, + "margin_dpo/margin_mean": 160.52728271484375, + "margin_dpo/margin_std": 193.41265869140625, + "step": 306 + }, + { + "KL/chosen_KL_mean": -221.96429443359375, + "KL/mean": -302.71917724609375, + "KL/rejected_KL_mean": -383.4740295410156, + "KL/std": 168.5135498046875, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.002254482125863433, + "fcm_dpo/delta": 0.03722835332155228, + "fcm_dpo/margin": 161.50973510742188, + "fcm_dpo/q_t": 0.41511091589927673, + "grad_norm": 27.366796493530273, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.4017038941383362, + "logits/rejected": -0.3805745840072632, + "logps/chosen": -292.12969970703125, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -470.44635009765625, + "loss": 1.107, + "margin_dpo/margin_mean": 161.50973510742188, + "margin_dpo/margin_std": 208.23245239257812, + "step": 307 + }, + { + "KL/chosen_KL_mean": -232.1256103515625, + "KL/mean": -323.47418212890625, + "KL/rejected_KL_mean": -414.8227233886719, + "KL/std": 194.7430419921875, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.0022605378180742264, + "fcm_dpo/delta": -0.013556101359426975, + "fcm_dpo/margin": 182.69711303710938, + "fcm_dpo/q_t": 0.4024621248245239, + "grad_norm": 35.28782653808594, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.3989991247653961, + "logits/rejected": -0.387599378824234, + "logps/chosen": -287.37060546875, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -494.19500732421875, + "loss": 1.0606, + "margin_dpo/margin_mean": 182.69711303710938, + "margin_dpo/margin_std": 191.560546875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -244.51980590820312, + "KL/mean": -340.3919677734375, + "KL/rejected_KL_mean": -436.2641296386719, + "KL/std": 215.52996826171875, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0022562372032552958, + "fcm_dpo/delta": -0.034387536346912384, + "fcm_dpo/margin": 191.74435424804688, + "fcm_dpo/q_t": 0.3991791009902954, + "grad_norm": 31.156871795654297, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.38808345794677734, + "logits/rejected": -0.3825611472129822, + "logps/chosen": -293.4788818359375, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -518.6048583984375, + "loss": 1.0668, + "margin_dpo/margin_mean": 191.74435424804688, + "margin_dpo/margin_std": 233.0519256591797, + "step": 309 + }, + { + "KL/chosen_KL_mean": -291.47320556640625, + "KL/mean": -379.0272216796875, + "KL/rejected_KL_mean": -466.581298828125, + "KL/std": 191.26611328125, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.002251605037599802, + "fcm_dpo/delta": 0.005678676068782806, + "fcm_dpo/margin": 175.10806274414062, + "fcm_dpo/q_t": 0.4070312976837158, + "grad_norm": 30.661561965942383, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.411726176738739, + "logits/rejected": -0.3994148373603821, + "logps/chosen": -354.2149658203125, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -546.5115966796875, + "loss": 1.0757, + "margin_dpo/margin_mean": 175.10806274414062, + "margin_dpo/margin_std": 189.48841857910156, + "step": 310 + }, + { + "KL/chosen_KL_mean": -300.638671875, + "KL/mean": -400.5716552734375, + "KL/rejected_KL_mean": -500.504638671875, + "KL/std": 247.5067138671875, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.0022279042750597, + "fcm_dpo/delta": -0.04740452021360397, + "fcm_dpo/margin": 199.865966796875, + "fcm_dpo/q_t": 0.39948105812072754, + "grad_norm": 27.384000778198242, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.3934406340122223, + "logits/rejected": -0.36774057149887085, + "logps/chosen": -353.6666564941406, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -577.94287109375, + "loss": 1.0678, + "margin_dpo/margin_mean": 199.865966796875, + "margin_dpo/margin_std": 262.3172912597656, + "step": 311 + }, + { + "KL/chosen_KL_mean": -284.6362609863281, + "KL/mean": -375.31683349609375, + "KL/rejected_KL_mean": -465.9974365234375, + "KL/std": 214.12132263183594, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0022299825213849545, + "fcm_dpo/delta": -0.004902999848127365, + "fcm_dpo/margin": 181.36117553710938, + "fcm_dpo/q_t": 0.4090193510055542, + "grad_norm": 26.90236473083496, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.35975557565689087, + "logits/rejected": -0.3475813865661621, + "logps/chosen": -334.0284729003906, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -541.7902221679688, + "loss": 1.0953, + "margin_dpo/margin_mean": 181.36117553710938, + "margin_dpo/margin_std": 254.2998046875, + "step": 312 + }, + { + "KL/chosen_KL_mean": -261.5609436035156, + "KL/mean": -353.8419494628906, + "KL/rejected_KL_mean": -446.1229553222656, + "KL/std": 220.43212890625, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.002231776248663664, + "fcm_dpo/delta": -0.01328357681632042, + "fcm_dpo/margin": 184.56201171875, + "fcm_dpo/q_t": 0.4050440788269043, + "grad_norm": 25.665958404541016, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.4496912956237793, + "logits/rejected": -0.4472866654396057, + "logps/chosen": -311.71368408203125, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -532.5291748046875, + "loss": 1.0888, + "margin_dpo/margin_mean": 184.56201171875, + "margin_dpo/margin_std": 242.4785919189453, + "step": 313 + }, + { + "KL/chosen_KL_mean": -274.48846435546875, + "KL/mean": -361.12188720703125, + "KL/rejected_KL_mean": -447.75537109375, + "KL/std": 211.32846069335938, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0022113409359008074, + "fcm_dpo/delta": 0.01723015308380127, + "fcm_dpo/margin": 173.26690673828125, + "fcm_dpo/q_t": 0.4142987132072449, + "grad_norm": 20.959877014160156, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.4525967836380005, + "logits/rejected": -0.4540666341781616, + "logps/chosen": -331.7260437011719, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -545.3519287109375, + "loss": 1.1226, + "margin_dpo/margin_mean": 173.26687622070312, + "margin_dpo/margin_std": 269.50933837890625, + "step": 314 + }, + { + "KL/chosen_KL_mean": -251.5740966796875, + "KL/mean": -331.55462646484375, + "KL/rejected_KL_mean": -411.53515625, + "KL/std": 184.97848510742188, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.0022343178279697895, + "fcm_dpo/delta": 0.04419757425785065, + "fcm_dpo/margin": 159.96112060546875, + "fcm_dpo/q_t": 0.41725191473960876, + "grad_norm": 22.308284759521484, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.3622434735298157, + "logits/rejected": -0.3640654981136322, + "logps/chosen": -300.6436767578125, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -497.216064453125, + "loss": 1.1117, + "margin_dpo/margin_mean": 159.96112060546875, + "margin_dpo/margin_std": 208.87994384765625, + "step": 315 + }, + { + "KL/chosen_KL_mean": -248.23333740234375, + "KL/mean": -361.19580078125, + "KL/rejected_KL_mean": -474.1582946777344, + "KL/std": 222.86338806152344, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0021978095173835754, + "fcm_dpo/delta": -0.10254909098148346, + "fcm_dpo/margin": 225.92495727539062, + "fcm_dpo/q_t": 0.38670068979263306, + "grad_norm": 27.076662063598633, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.4021075367927551, + "logits/rejected": -0.40691736340522766, + "logps/chosen": -302.49407958984375, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -575.4397583007812, + "loss": 1.0176, + "margin_dpo/margin_mean": 225.92495727539062, + "margin_dpo/margin_std": 246.41647338867188, + "step": 316 + }, + { + "KL/chosen_KL_mean": -253.09286499023438, + "KL/mean": -376.51629638671875, + "KL/rejected_KL_mean": -499.9396667480469, + "KL/std": 205.02980041503906, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.002158908173441887, + "fcm_dpo/delta": -0.140162855386734, + "fcm_dpo/margin": 246.8468017578125, + "fcm_dpo/q_t": 0.37586137652397156, + "grad_norm": 32.17722702026367, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.3921947777271271, + "logits/rejected": -0.38066309690475464, + "logps/chosen": -309.18707275390625, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -600.6387329101562, + "loss": 0.9814, + "margin_dpo/margin_mean": 246.84677124023438, + "margin_dpo/margin_std": 227.67401123046875, + "step": 317 + }, + { + "KL/chosen_KL_mean": -272.281005859375, + "KL/mean": -365.32550048828125, + "KL/rejected_KL_mean": -458.36993408203125, + "KL/std": 213.32907104492188, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0021421005949378014, + "fcm_dpo/delta": 0.0012923870235681534, + "fcm_dpo/margin": 186.08895874023438, + "fcm_dpo/q_t": 0.4077012538909912, + "grad_norm": 28.045991897583008, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.4386771023273468, + "logits/rejected": -0.4078526496887207, + "logps/chosen": -336.92669677734375, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -541.1341552734375, + "loss": 1.0836, + "margin_dpo/margin_mean": 186.08895874023438, + "margin_dpo/margin_std": 229.52560424804688, + "step": 318 + }, + { + "KL/chosen_KL_mean": -245.36065673828125, + "KL/mean": -353.22686767578125, + "KL/rejected_KL_mean": -461.09307861328125, + "KL/std": 227.05068969726562, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.0021182619966566563, + "fcm_dpo/delta": -0.059692900627851486, + "fcm_dpo/margin": 215.73245239257812, + "fcm_dpo/q_t": 0.39378929138183594, + "grad_norm": 23.20696449279785, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.3748522400856018, + "logits/rejected": -0.38931792974472046, + "logps/chosen": -294.744384765625, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -574.9996337890625, + "loss": 1.0437, + "margin_dpo/margin_mean": 215.73245239257812, + "margin_dpo/margin_std": 246.70590209960938, + "step": 319 + }, + { + "KL/chosen_KL_mean": -258.44439697265625, + "KL/mean": -378.5030822753906, + "KL/rejected_KL_mean": -498.561767578125, + "KL/std": 238.49586486816406, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.00207329448312521, + "fcm_dpo/delta": -0.10340476781129837, + "fcm_dpo/margin": 240.11732482910156, + "fcm_dpo/q_t": 0.38526278734207153, + "grad_norm": 29.931739807128906, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.41057881712913513, + "logits/rejected": -0.4162050783634186, + "logps/chosen": -317.94927978515625, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -596.2288818359375, + "loss": 1.0125, + "margin_dpo/margin_mean": 240.11732482910156, + "margin_dpo/margin_std": 254.24203491210938, + "step": 320 + }, + { + "KL/chosen_KL_mean": -320.3726501464844, + "KL/mean": -421.73480224609375, + "KL/rejected_KL_mean": -523.0969848632812, + "KL/std": 244.77264404296875, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.002050665207207203, + "fcm_dpo/delta": -0.017213057726621628, + "fcm_dpo/margin": 202.72438049316406, + "fcm_dpo/q_t": 0.4042757451534271, + "grad_norm": 23.253496170043945, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.3695087134838104, + "logits/rejected": -0.360482394695282, + "logps/chosen": -381.92132568359375, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -614.738037109375, + "loss": 1.0825, + "margin_dpo/margin_mean": 202.724365234375, + "margin_dpo/margin_std": 259.14007568359375, + "step": 321 + }, + { + "KL/chosen_KL_mean": -281.20330810546875, + "KL/mean": -396.8486328125, + "KL/rejected_KL_mean": -512.493896484375, + "KL/std": 223.7625732421875, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.002031845971941948, + "fcm_dpo/delta": -0.07384911924600601, + "fcm_dpo/margin": 231.29055786132812, + "fcm_dpo/q_t": 0.39028504490852356, + "grad_norm": 23.78490447998047, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.3746291995048523, + "logits/rejected": -0.3742471933364868, + "logps/chosen": -338.49334716796875, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -608.2438354492188, + "loss": 1.0264, + "margin_dpo/margin_mean": 231.29055786132812, + "margin_dpo/margin_std": 238.2588348388672, + "step": 322 + }, + { + "KL/chosen_KL_mean": -307.2816162109375, + "KL/mean": -406.22393798828125, + "KL/rejected_KL_mean": -505.16619873046875, + "KL/std": 223.4599609375, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.0020272608380764723, + "fcm_dpo/delta": -0.0012617167085409164, + "fcm_dpo/margin": 197.8845977783203, + "fcm_dpo/q_t": 0.40663379430770874, + "grad_norm": 30.926660537719727, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.4343390464782715, + "logits/rejected": -0.4323709309101105, + "logps/chosen": -358.51556396484375, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -580.2281494140625, + "loss": 1.0853, + "margin_dpo/margin_mean": 197.8845977783203, + "margin_dpo/margin_std": 243.69021606445312, + "step": 323 + }, + { + "KL/chosen_KL_mean": -356.24090576171875, + "KL/mean": -450.4142150878906, + "KL/rejected_KL_mean": -544.5875244140625, + "KL/std": 240.1791229248047, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.0020489755552262068, + "fcm_dpo/delta": 0.013504624366760254, + "fcm_dpo/margin": 188.3466033935547, + "fcm_dpo/q_t": 0.41284650564193726, + "grad_norm": 39.6912841796875, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.4159066677093506, + "logits/rejected": -0.4032629132270813, + "logps/chosen": -421.3760986328125, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -631.0650024414062, + "loss": 1.1161, + "margin_dpo/margin_mean": 188.3466033935547, + "margin_dpo/margin_std": 269.37054443359375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -281.6850891113281, + "KL/mean": -360.40020751953125, + "KL/rejected_KL_mean": -439.1153564453125, + "KL/std": 206.2490997314453, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.002061846200376749, + "fcm_dpo/delta": 0.07781701534986496, + "fcm_dpo/margin": 157.43028259277344, + "fcm_dpo/q_t": 0.4237811267375946, + "grad_norm": 26.56155776977539, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.48065128922462463, + "logits/rejected": -0.46600067615509033, + "logps/chosen": -337.90069580078125, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -509.2012939453125, + "loss": 1.1334, + "margin_dpo/margin_mean": 157.4302978515625, + "margin_dpo/margin_std": 205.68899536132812, + "step": 325 + }, + { + "KL/chosen_KL_mean": -318.42431640625, + "KL/mean": -392.8778381347656, + "KL/rejected_KL_mean": -467.33135986328125, + "KL/std": 199.95083618164062, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0020911036990582943, + "fcm_dpo/delta": 0.09157069027423859, + "fcm_dpo/margin": 148.90704345703125, + "fcm_dpo/q_t": 0.42620426416397095, + "grad_norm": 49.48163604736328, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.4975377917289734, + "logits/rejected": -0.4879492521286011, + "logps/chosen": -391.1492919921875, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -547.1781616210938, + "loss": 1.14, + "margin_dpo/margin_mean": 148.90704345703125, + "margin_dpo/margin_std": 190.74583435058594, + "step": 326 + }, + { + "KL/chosen_KL_mean": -279.2503967285156, + "KL/mean": -390.81011962890625, + "KL/rejected_KL_mean": -502.369873046875, + "KL/std": 209.59689331054688, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0020797294564545155, + "fcm_dpo/delta": -0.06743220239877701, + "fcm_dpo/margin": 223.11947631835938, + "fcm_dpo/q_t": 0.3905792534351349, + "grad_norm": 47.27796936035156, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.471914142370224, + "logits/rejected": -0.46862471103668213, + "logps/chosen": -348.38482666015625, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -614.3036499023438, + "loss": 1.0264, + "margin_dpo/margin_mean": 223.11947631835938, + "margin_dpo/margin_std": 223.48355102539062, + "step": 327 + }, + { + "KL/chosen_KL_mean": -299.3519287109375, + "KL/mean": -404.4681396484375, + "KL/rejected_KL_mean": -509.5843200683594, + "KL/std": 233.9372100830078, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0020683803595602512, + "fcm_dpo/delta": -0.036408208310604095, + "fcm_dpo/margin": 210.23236083984375, + "fcm_dpo/q_t": 0.400495707988739, + "grad_norm": 36.852413177490234, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.45196738839149475, + "logits/rejected": -0.4595873951911926, + "logps/chosen": -359.03912353515625, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -600.4393310546875, + "loss": 1.0657, + "margin_dpo/margin_mean": 210.2323760986328, + "margin_dpo/margin_std": 263.08758544921875, + "step": 328 + }, + { + "KL/chosen_KL_mean": -344.8146057128906, + "KL/mean": -453.81494140625, + "KL/rejected_KL_mean": -562.8153076171875, + "KL/std": 255.34573364257812, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.002038725186139345, + "fcm_dpo/delta": -0.04716340824961662, + "fcm_dpo/margin": 218.00067138671875, + "fcm_dpo/q_t": 0.39805126190185547, + "grad_norm": 31.667083740234375, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.4629364609718323, + "logits/rejected": -0.4617495834827423, + "logps/chosen": -410.060791015625, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -663.5130004882812, + "loss": 1.0698, + "margin_dpo/margin_mean": 218.00067138671875, + "margin_dpo/margin_std": 279.3451843261719, + "step": 329 + }, + { + "KL/chosen_KL_mean": -294.402587890625, + "KL/mean": -411.0, + "KL/rejected_KL_mean": -527.597412109375, + "KL/std": 250.00555419921875, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0020207413472235203, + "fcm_dpo/delta": -0.07474477589130402, + "fcm_dpo/margin": 233.19488525390625, + "fcm_dpo/q_t": 0.3922659456729889, + "grad_norm": 25.003374099731445, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.41176387667655945, + "logits/rejected": -0.41329875588417053, + "logps/chosen": -341.4009094238281, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -614.4743041992188, + "loss": 1.0401, + "margin_dpo/margin_mean": 233.19488525390625, + "margin_dpo/margin_std": 275.0955810546875, + "step": 330 + }, + { + "KL/chosen_KL_mean": -315.91162109375, + "KL/mean": -432.8357849121094, + "KL/rejected_KL_mean": -549.760009765625, + "KL/std": 209.0313720703125, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.001991886179894209, + "fcm_dpo/delta": -0.06903053820133209, + "fcm_dpo/margin": 233.84837341308594, + "fcm_dpo/q_t": 0.3900572955608368, + "grad_norm": 26.756946563720703, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.42531535029411316, + "logits/rejected": -0.42004719376564026, + "logps/chosen": -366.43585205078125, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -638.775390625, + "loss": 1.0133, + "margin_dpo/margin_mean": 233.84837341308594, + "margin_dpo/margin_std": 208.65243530273438, + "step": 331 + }, + { + "KL/chosen_KL_mean": -310.6379089355469, + "KL/mean": -394.1624755859375, + "KL/rejected_KL_mean": -477.68701171875, + "KL/std": 215.14508056640625, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.0020075312349945307, + "fcm_dpo/delta": 0.06678016483783722, + "fcm_dpo/margin": 167.04910278320312, + "fcm_dpo/q_t": 0.42322736978530884, + "grad_norm": 25.563440322875977, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.43550896644592285, + "logits/rejected": -0.4156872630119324, + "logps/chosen": -359.81817626953125, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -554.1721801757812, + "loss": 1.1332, + "margin_dpo/margin_mean": 167.04910278320312, + "margin_dpo/margin_std": 241.10047912597656, + "step": 332 + }, + { + "KL/chosen_KL_mean": -328.4854736328125, + "KL/mean": -434.61962890625, + "KL/rejected_KL_mean": -540.7538452148438, + "KL/std": 250.06927490234375, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.0019904375076293945, + "fcm_dpo/delta": -0.02470763772726059, + "fcm_dpo/margin": 212.26832580566406, + "fcm_dpo/q_t": 0.4042668342590332, + "grad_norm": 26.71259117126465, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.4605112075805664, + "logits/rejected": -0.4544973075389862, + "logps/chosen": -392.2412109375, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -635.7979736328125, + "loss": 1.0951, + "margin_dpo/margin_mean": 212.26832580566406, + "margin_dpo/margin_std": 299.41314697265625, + "step": 333 + }, + { + "KL/chosen_KL_mean": -313.60162353515625, + "KL/mean": -409.2901611328125, + "KL/rejected_KL_mean": -504.9786682128906, + "KL/std": 270.18035888671875, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0020047123543918133, + "fcm_dpo/delta": 0.016986673697829247, + "fcm_dpo/margin": 191.37701416015625, + "fcm_dpo/q_t": 0.41407567262649536, + "grad_norm": 27.721216201782227, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.46878981590270996, + "logits/rejected": -0.4665107727050781, + "logps/chosen": -380.5813903808594, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -600.2955932617188, + "loss": 1.1295, + "margin_dpo/margin_mean": 191.3770294189453, + "margin_dpo/margin_std": 309.3908386230469, + "step": 334 + }, + { + "KL/chosen_KL_mean": -332.8387756347656, + "KL/mean": -409.174072265625, + "KL/rejected_KL_mean": -485.5093994140625, + "KL/std": 250.36761474609375, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.002047334797680378, + "fcm_dpo/delta": 0.08909609913825989, + "fcm_dpo/margin": 152.67063903808594, + "fcm_dpo/q_t": 0.4297252595424652, + "grad_norm": 28.045196533203125, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.49084708094596863, + "logits/rejected": -0.4701639413833618, + "logps/chosen": -395.3812561035156, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -573.1270751953125, + "loss": 1.1715, + "margin_dpo/margin_mean": 152.67063903808594, + "margin_dpo/margin_std": 267.66217041015625, + "step": 335 + }, + { + "KL/chosen_KL_mean": -336.10711669921875, + "KL/mean": -456.0296630859375, + "KL/rejected_KL_mean": -575.9522705078125, + "KL/std": 288.1374206542969, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.002026779344305396, + "fcm_dpo/delta": -0.09051845967769623, + "fcm_dpo/margin": 239.84512329101562, + "fcm_dpo/q_t": 0.3925360441207886, + "grad_norm": 29.927021026611328, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.43909382820129395, + "logits/rejected": -0.437904417514801, + "logps/chosen": -390.63824462890625, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -680.3564453125, + "loss": 1.0708, + "margin_dpo/margin_mean": 239.84512329101562, + "margin_dpo/margin_std": 344.2579650878906, + "step": 336 + }, + { + "KL/chosen_KL_mean": -303.54913330078125, + "KL/mean": -423.3258972167969, + "KL/rejected_KL_mean": -543.1026611328125, + "KL/std": 232.45880126953125, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.0019804544281214476, + "fcm_dpo/delta": -0.07842068374156952, + "fcm_dpo/margin": 239.55349731445312, + "fcm_dpo/q_t": 0.3914318382740021, + "grad_norm": 29.8690128326416, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.4820428192615509, + "logits/rejected": -0.4706416726112366, + "logps/chosen": -368.6778259277344, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -644.8296508789062, + "loss": 1.0357, + "margin_dpo/margin_mean": 239.55349731445312, + "margin_dpo/margin_std": 277.24346923828125, + "step": 337 + }, + { + "KL/chosen_KL_mean": -270.0802917480469, + "KL/mean": -384.84307861328125, + "KL/rejected_KL_mean": -499.6058349609375, + "KL/std": 217.44955444335938, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.001958791632205248, + "fcm_dpo/delta": -0.05205549672245979, + "fcm_dpo/margin": 229.52557373046875, + "fcm_dpo/q_t": 0.39443230628967285, + "grad_norm": 30.42357063293457, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.48334747552871704, + "logits/rejected": -0.476375937461853, + "logps/chosen": -328.50299072265625, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -588.6743774414062, + "loss": 1.034, + "margin_dpo/margin_mean": 229.52557373046875, + "margin_dpo/margin_std": 230.23306274414062, + "step": 338 + }, + { + "KL/chosen_KL_mean": -290.23211669921875, + "KL/mean": -402.99951171875, + "KL/rejected_KL_mean": -515.7669677734375, + "KL/std": 229.53768920898438, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0019593099132180214, + "fcm_dpo/delta": -0.04478244110941887, + "fcm_dpo/margin": 225.5348358154297, + "fcm_dpo/q_t": 0.39693379402160645, + "grad_norm": 24.601205825805664, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.5301988124847412, + "logits/rejected": -0.5157142877578735, + "logps/chosen": -350.2274169921875, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -619.6778564453125, + "loss": 1.0478, + "margin_dpo/margin_mean": 225.5348358154297, + "margin_dpo/margin_std": 232.717529296875, + "step": 339 + }, + { + "KL/chosen_KL_mean": -302.79833984375, + "KL/mean": -401.8282775878906, + "KL/rejected_KL_mean": -500.8582763671875, + "KL/std": 219.39871215820312, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0019372242968529463, + "fcm_dpo/delta": 0.01677379384636879, + "fcm_dpo/margin": 198.05987548828125, + "fcm_dpo/q_t": 0.4115055799484253, + "grad_norm": 27.53554344177246, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.4480810761451721, + "logits/rejected": -0.42684268951416016, + "logps/chosen": -355.6285705566406, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -573.9654541015625, + "loss": 1.1038, + "margin_dpo/margin_mean": 198.05987548828125, + "margin_dpo/margin_std": 269.5626220703125, + "step": 340 + }, + { + "KL/chosen_KL_mean": -300.7313232421875, + "KL/mean": -412.8475036621094, + "KL/rejected_KL_mean": -524.9636840820312, + "KL/std": 240.1968994140625, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0019282340072095394, + "fcm_dpo/delta": -0.03427855670452118, + "fcm_dpo/margin": 224.23236083984375, + "fcm_dpo/q_t": 0.40013015270233154, + "grad_norm": 24.84458351135254, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.4511696398258209, + "logits/rejected": -0.4502286911010742, + "logps/chosen": -348.631103515625, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -626.7735595703125, + "loss": 1.0591, + "margin_dpo/margin_mean": 224.23236083984375, + "margin_dpo/margin_std": 256.24444580078125, + "step": 341 + }, + { + "KL/chosen_KL_mean": -301.24267578125, + "KL/mean": -401.7803955078125, + "KL/rejected_KL_mean": -502.318115234375, + "KL/std": 223.74557495117188, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0019417132716625929, + "fcm_dpo/delta": 0.009692085906863213, + "fcm_dpo/margin": 201.07542419433594, + "fcm_dpo/q_t": 0.40966731309890747, + "grad_norm": 26.8356876373291, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.47591298818588257, + "logits/rejected": -0.4502405524253845, + "logps/chosen": -373.23931884765625, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -594.90771484375, + "loss": 1.0941, + "margin_dpo/margin_mean": 201.07540893554688, + "margin_dpo/margin_std": 261.2144470214844, + "step": 342 + }, + { + "KL/chosen_KL_mean": -294.72625732421875, + "KL/mean": -413.42022705078125, + "KL/rejected_KL_mean": -532.1141357421875, + "KL/std": 228.767578125, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.001919258153066039, + "fcm_dpo/delta": -0.05826767161488533, + "fcm_dpo/margin": 237.387939453125, + "fcm_dpo/q_t": 0.3916386663913727, + "grad_norm": 24.52044677734375, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.4904317259788513, + "logits/rejected": -0.495491087436676, + "logps/chosen": -349.1318664550781, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -643.1556396484375, + "loss": 1.0194, + "margin_dpo/margin_mean": 237.387939453125, + "margin_dpo/margin_std": 202.62887573242188, + "step": 343 + }, + { + "KL/chosen_KL_mean": -295.74444580078125, + "KL/mean": -399.68359375, + "KL/rejected_KL_mean": -503.62274169921875, + "KL/std": 247.79766845703125, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.0019302130676805973, + "fcm_dpo/delta": -0.0029281899333000183, + "fcm_dpo/margin": 207.8782958984375, + "fcm_dpo/q_t": 0.4088844060897827, + "grad_norm": 32.13612747192383, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.4950554072856903, + "logits/rejected": -0.5000725388526917, + "logps/chosen": -349.7091064453125, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -594.24609375, + "loss": 1.0867, + "margin_dpo/margin_mean": 207.8782958984375, + "margin_dpo/margin_std": 252.4293975830078, + "step": 344 + }, + { + "KL/chosen_KL_mean": -350.2357177734375, + "KL/mean": -465.2918701171875, + "KL/rejected_KL_mean": -580.3480224609375, + "KL/std": 237.791259765625, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0019050377886742353, + "fcm_dpo/delta": -0.04013466089963913, + "fcm_dpo/margin": 230.11233520507812, + "fcm_dpo/q_t": 0.398155152797699, + "grad_norm": 23.150665283203125, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.4910598397254944, + "logits/rejected": -0.48923730850219727, + "logps/chosen": -411.92144775390625, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -679.8384399414062, + "loss": 1.0554, + "margin_dpo/margin_mean": 230.11231994628906, + "margin_dpo/margin_std": 266.40472412109375, + "step": 345 + }, + { + "KL/chosen_KL_mean": -348.7013244628906, + "KL/mean": -459.8345947265625, + "KL/rejected_KL_mean": -570.9678955078125, + "KL/std": 235.310302734375, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.0018911452498286963, + "fcm_dpo/delta": -0.02123241126537323, + "fcm_dpo/margin": 222.26657104492188, + "fcm_dpo/q_t": 0.4010668396949768, + "grad_norm": 20.939437866210938, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.4846650958061218, + "logits/rejected": -0.47431010007858276, + "logps/chosen": -403.95758056640625, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -648.3831787109375, + "loss": 1.0619, + "margin_dpo/margin_mean": 222.26658630371094, + "margin_dpo/margin_std": 249.186279296875, + "step": 346 + }, + { + "KL/chosen_KL_mean": -349.28948974609375, + "KL/mean": -456.8357238769531, + "KL/rejected_KL_mean": -564.3819580078125, + "KL/std": 246.46975708007812, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.0018879894632846117, + "fcm_dpo/delta": -0.006350051611661911, + "fcm_dpo/margin": 215.09246826171875, + "fcm_dpo/q_t": 0.4059875011444092, + "grad_norm": 21.86200523376465, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.4439418315887451, + "logits/rejected": -0.4371364712715149, + "logps/chosen": -406.8557434082031, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -656.737060546875, + "loss": 1.081, + "margin_dpo/margin_mean": 215.09246826171875, + "margin_dpo/margin_std": 268.1534729003906, + "step": 347 + }, + { + "KL/chosen_KL_mean": -318.54083251953125, + "KL/mean": -410.75787353515625, + "KL/rejected_KL_mean": -502.9749755859375, + "KL/std": 212.30099487304688, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.0018904341850429773, + "fcm_dpo/delta": 0.05277785286307335, + "fcm_dpo/margin": 184.43411254882812, + "fcm_dpo/q_t": 0.4192753732204437, + "grad_norm": 21.335830688476562, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.4899284243583679, + "logits/rejected": -0.49212366342544556, + "logps/chosen": -374.8585205078125, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -592.1133422851562, + "loss": 1.1219, + "margin_dpo/margin_mean": 184.43411254882812, + "margin_dpo/margin_std": 245.26687622070312, + "step": 348 + }, + { + "KL/chosen_KL_mean": -307.7067565917969, + "KL/mean": -414.20538330078125, + "KL/rejected_KL_mean": -520.7040405273438, + "KL/std": 242.98814392089844, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.0019131149165332317, + "fcm_dpo/delta": -0.00818055123090744, + "fcm_dpo/margin": 212.9972381591797, + "fcm_dpo/q_t": 0.40792983770370483, + "grad_norm": 22.361181259155273, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.48233842849731445, + "logits/rejected": -0.48526209592819214, + "logps/chosen": -365.7322692871094, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -618.2091674804688, + "loss": 1.0952, + "margin_dpo/margin_mean": 212.9972381591797, + "margin_dpo/margin_std": 296.6618957519531, + "step": 349 + }, + { + "KL/chosen_KL_mean": -328.4967041015625, + "KL/mean": -434.0699768066406, + "KL/rejected_KL_mean": -539.6432495117188, + "KL/std": 228.32424926757812, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.001895663677714765, + "fcm_dpo/delta": -0.0006302520632743835, + "fcm_dpo/margin": 211.14651489257812, + "fcm_dpo/q_t": 0.4063173532485962, + "grad_norm": 32.4691162109375, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.4853939414024353, + "logits/rejected": -0.4879404306411743, + "logps/chosen": -392.82720947265625, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -629.514892578125, + "loss": 1.093, + "margin_dpo/margin_mean": 211.14654541015625, + "margin_dpo/margin_std": 278.280517578125, + "step": 350 + }, + { + "KL/chosen_KL_mean": -324.23406982421875, + "KL/mean": -446.6317443847656, + "KL/rejected_KL_mean": -569.0294189453125, + "KL/std": 273.3016662597656, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0018817733507603407, + "fcm_dpo/delta": -0.06378539651632309, + "fcm_dpo/margin": 244.79534912109375, + "fcm_dpo/q_t": 0.3960220217704773, + "grad_norm": 23.294527053833008, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.4951154291629791, + "logits/rejected": -0.493879497051239, + "logps/chosen": -384.90618896484375, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -670.5948486328125, + "loss": 1.0584, + "margin_dpo/margin_mean": 244.79534912109375, + "margin_dpo/margin_std": 315.10546875, + "step": 351 + }, + { + "KL/chosen_KL_mean": -380.00848388671875, + "KL/mean": -462.681396484375, + "KL/rejected_KL_mean": -545.3543701171875, + "KL/std": 250.68218994140625, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.0018972533289343119, + "fcm_dpo/delta": 0.08913286030292511, + "fcm_dpo/margin": 165.34588623046875, + "fcm_dpo/q_t": 0.42799216508865356, + "grad_norm": 34.53774642944336, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.51950603723526, + "logits/rejected": -0.48839718103408813, + "logps/chosen": -450.951904296875, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -621.996337890625, + "loss": 1.1609, + "margin_dpo/margin_mean": 165.34588623046875, + "margin_dpo/margin_std": 269.165283203125, + "step": 352 + }, + { + "KL/chosen_KL_mean": -357.87823486328125, + "KL/mean": -464.0823974609375, + "KL/rejected_KL_mean": -570.2864990234375, + "KL/std": 257.20391845703125, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.001901305397041142, + "fcm_dpo/delta": -0.004667513072490692, + "fcm_dpo/margin": 212.40826416015625, + "fcm_dpo/q_t": 0.4078369438648224, + "grad_norm": 26.698135375976562, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.5590090751647949, + "logits/rejected": -0.546757161617279, + "logps/chosen": -416.2735595703125, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -650.6220703125, + "loss": 1.0974, + "margin_dpo/margin_mean": 212.40826416015625, + "margin_dpo/margin_std": 291.78887939453125, + "step": 353 + }, + { + "KL/chosen_KL_mean": -307.4017333984375, + "KL/mean": -426.0465087890625, + "KL/rejected_KL_mean": -544.6912231445312, + "KL/std": 233.544921875, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.0018950920784845948, + "fcm_dpo/delta": -0.05208052694797516, + "fcm_dpo/margin": 237.2895050048828, + "fcm_dpo/q_t": 0.3941301703453064, + "grad_norm": 26.772897720336914, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.5388429760932922, + "logits/rejected": -0.5311779975891113, + "logps/chosen": -367.2047119140625, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -633.44873046875, + "loss": 1.0301, + "margin_dpo/margin_mean": 237.2895050048828, + "margin_dpo/margin_std": 228.05111694335938, + "step": 354 + }, + { + "KL/chosen_KL_mean": -306.6812438964844, + "KL/mean": -421.31683349609375, + "KL/rejected_KL_mean": -535.952392578125, + "KL/std": 237.99423217773438, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0018783884588629007, + "fcm_dpo/delta": -0.0322100892663002, + "fcm_dpo/margin": 229.27114868164062, + "fcm_dpo/q_t": 0.3988185524940491, + "grad_norm": 35.09297561645508, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.5958764553070068, + "logits/rejected": -0.5878010988235474, + "logps/chosen": -360.80975341796875, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -618.3584594726562, + "loss": 1.0506, + "margin_dpo/margin_mean": 229.27114868164062, + "margin_dpo/margin_std": 241.91305541992188, + "step": 355 + }, + { + "KL/chosen_KL_mean": -368.34222412109375, + "KL/mean": -431.9561767578125, + "KL/rejected_KL_mean": -495.57012939453125, + "KL/std": 253.87655639648438, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.001934206229634583, + "fcm_dpo/delta": 0.15637606382369995, + "fcm_dpo/margin": 127.22789764404297, + "fcm_dpo/q_t": 0.44535139203071594, + "grad_norm": 39.2513542175293, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.5445902943611145, + "logits/rejected": -0.5229723453521729, + "logps/chosen": -433.01605224609375, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -571.4693603515625, + "loss": 1.246, + "margin_dpo/margin_mean": 127.2279052734375, + "margin_dpo/margin_std": 317.629150390625, + "step": 356 + }, + { + "KL/chosen_KL_mean": -329.9525146484375, + "KL/mean": -433.06060791015625, + "KL/rejected_KL_mean": -536.168701171875, + "KL/std": 243.35272216796875, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.0019303904846310616, + "fcm_dpo/delta": 0.0015458688139915466, + "fcm_dpo/margin": 206.21620178222656, + "fcm_dpo/q_t": 0.40688982605934143, + "grad_norm": 29.09072494506836, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.5348184108734131, + "logits/rejected": -0.5215445756912231, + "logps/chosen": -382.67828369140625, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -623.0098876953125, + "loss": 1.0849, + "margin_dpo/margin_mean": 206.21621704101562, + "margin_dpo/margin_std": 251.61915588378906, + "step": 357 + }, + { + "KL/chosen_KL_mean": -297.680908203125, + "KL/mean": -394.7061462402344, + "KL/rejected_KL_mean": -491.7313537597656, + "KL/std": 234.25167846679688, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.001946124481037259, + "fcm_dpo/delta": 0.023234549909830093, + "fcm_dpo/margin": 194.05039978027344, + "fcm_dpo/q_t": 0.4129307270050049, + "grad_norm": 27.710437774658203, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.5369811654090881, + "logits/rejected": -0.51755690574646, + "logps/chosen": -360.8863525390625, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -580.1046142578125, + "loss": 1.0989, + "margin_dpo/margin_mean": 194.05038452148438, + "margin_dpo/margin_std": 248.99649047851562, + "step": 358 + }, + { + "KL/chosen_KL_mean": -332.6034240722656, + "KL/mean": -446.4267883300781, + "KL/rejected_KL_mean": -560.2501220703125, + "KL/std": 236.38690185546875, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0019337392877787352, + "fcm_dpo/delta": -0.04216692969202995, + "fcm_dpo/margin": 227.646728515625, + "fcm_dpo/q_t": 0.39980757236480713, + "grad_norm": 31.509368896484375, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.5173889398574829, + "logits/rejected": -0.5052172541618347, + "logps/chosen": -388.9736328125, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -642.4238891601562, + "loss": 1.0663, + "margin_dpo/margin_mean": 227.646728515625, + "margin_dpo/margin_std": 286.3934326171875, + "step": 359 + }, + { + "KL/chosen_KL_mean": -314.5792236328125, + "KL/mean": -400.4184265136719, + "KL/rejected_KL_mean": -486.25762939453125, + "KL/std": 194.84371948242188, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.0019554642494767904, + "fcm_dpo/delta": 0.06631289422512054, + "fcm_dpo/margin": 171.6783447265625, + "fcm_dpo/q_t": 0.4207460284233093, + "grad_norm": 33.207435607910156, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.5018008947372437, + "logits/rejected": -0.48616838455200195, + "logps/chosen": -366.03961181640625, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -556.0965576171875, + "loss": 1.1154, + "margin_dpo/margin_mean": 171.6783447265625, + "margin_dpo/margin_std": 198.9645233154297, + "step": 360 + }, + { + "KL/chosen_KL_mean": -325.6524658203125, + "KL/mean": -412.54840087890625, + "KL/rejected_KL_mean": -499.44427490234375, + "KL/std": 230.0059051513672, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.001980352681130171, + "fcm_dpo/delta": 0.0575227327644825, + "fcm_dpo/margin": 173.79180908203125, + "fcm_dpo/q_t": 0.42029404640197754, + "grad_norm": 37.1341667175293, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.5017886161804199, + "logits/rejected": -0.49733275175094604, + "logps/chosen": -379.5220031738281, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -590.2135620117188, + "loss": 1.1321, + "margin_dpo/margin_mean": 173.79180908203125, + "margin_dpo/margin_std": 254.7476043701172, + "step": 361 + }, + { + "KL/chosen_KL_mean": -280.2534484863281, + "KL/mean": -413.67962646484375, + "KL/rejected_KL_mean": -547.1058349609375, + "KL/std": 240.3780517578125, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.0019390920642763376, + "fcm_dpo/delta": -0.12437668442726135, + "fcm_dpo/margin": 266.8523254394531, + "fcm_dpo/q_t": 0.3802497982978821, + "grad_norm": 28.240169525146484, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.5208746194839478, + "logits/rejected": -0.5305719375610352, + "logps/chosen": -338.89251708984375, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -652.687744140625, + "loss": 0.9903, + "margin_dpo/margin_mean": 266.852294921875, + "margin_dpo/margin_std": 257.30804443359375, + "step": 362 + }, + { + "KL/chosen_KL_mean": -247.235107421875, + "KL/mean": -383.9427490234375, + "KL/rejected_KL_mean": -520.650390625, + "KL/std": 235.0074462890625, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0019011329859495163, + "fcm_dpo/delta": -0.1261799931526184, + "fcm_dpo/margin": 273.415283203125, + "fcm_dpo/q_t": 0.3791302442550659, + "grad_norm": 30.7271785736084, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.48286134004592896, + "logits/rejected": -0.4701201915740967, + "logps/chosen": -291.7934875488281, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -595.3453369140625, + "loss": 0.9868, + "margin_dpo/margin_mean": 273.415283203125, + "margin_dpo/margin_std": 254.3519287109375, + "step": 363 + }, + { + "KL/chosen_KL_mean": -275.04034423828125, + "KL/mean": -377.3860778808594, + "KL/rejected_KL_mean": -479.7318420410156, + "KL/std": 228.88882446289062, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0018944879993796349, + "fcm_dpo/delta": 0.012386158108711243, + "fcm_dpo/margin": 204.6914520263672, + "fcm_dpo/q_t": 0.4097138047218323, + "grad_norm": 23.02882194519043, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.48808223009109497, + "logits/rejected": -0.4956515431404114, + "logps/chosen": -323.9349670410156, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -571.1275634765625, + "loss": 1.0942, + "margin_dpo/margin_mean": 204.69146728515625, + "margin_dpo/margin_std": 260.9207763671875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -268.84808349609375, + "KL/mean": -377.37420654296875, + "KL/rejected_KL_mean": -485.90032958984375, + "KL/std": 243.60565185546875, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0018870255444198847, + "fcm_dpo/delta": -0.009993639774620533, + "fcm_dpo/margin": 217.05224609375, + "fcm_dpo/q_t": 0.4056761860847473, + "grad_norm": 28.16781997680664, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.4856104254722595, + "logits/rejected": -0.49556827545166016, + "logps/chosen": -320.3408203125, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -578.6019897460938, + "loss": 1.0702, + "margin_dpo/margin_mean": 217.05224609375, + "margin_dpo/margin_std": 250.88876342773438, + "step": 365 + }, + { + "KL/chosen_KL_mean": -255.52041625976562, + "KL/mean": -366.74090576171875, + "KL/rejected_KL_mean": -477.96142578125, + "KL/std": 236.849853515625, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0018722948152571917, + "fcm_dpo/delta": -0.017610307782888412, + "fcm_dpo/margin": 222.44097900390625, + "fcm_dpo/q_t": 0.40416595339775085, + "grad_norm": 22.661331176757812, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.47054141759872437, + "logits/rejected": -0.48637819290161133, + "logps/chosen": -300.2409973144531, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -561.2718505859375, + "loss": 1.0777, + "margin_dpo/margin_mean": 222.4409637451172, + "margin_dpo/margin_std": 279.572265625, + "step": 366 + }, + { + "KL/chosen_KL_mean": -275.62713623046875, + "KL/mean": -367.9857177734375, + "KL/rejected_KL_mean": -460.34429931640625, + "KL/std": 215.36822509765625, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0018970149103552103, + "fcm_dpo/delta": 0.051168061792850494, + "fcm_dpo/margin": 184.71719360351562, + "fcm_dpo/q_t": 0.4176059663295746, + "grad_norm": 21.282058715820312, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.46719565987586975, + "logits/rejected": -0.44931039214134216, + "logps/chosen": -334.03253173828125, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -537.0956420898438, + "loss": 1.1141, + "margin_dpo/margin_mean": 184.71719360351562, + "margin_dpo/margin_std": 234.90869140625, + "step": 367 + }, + { + "KL/chosen_KL_mean": -262.6880798339844, + "KL/mean": -408.541015625, + "KL/rejected_KL_mean": -554.3939208984375, + "KL/std": 247.9871826171875, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0018567652441561222, + "fcm_dpo/delta": -0.1496785581111908, + "fcm_dpo/margin": 291.70587158203125, + "fcm_dpo/q_t": 0.3715639114379883, + "grad_norm": 34.26023483276367, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.4799085855484009, + "logits/rejected": -0.48711296916007996, + "logps/chosen": -307.1405944824219, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -652.94921875, + "loss": 0.9574, + "margin_dpo/margin_mean": 291.70587158203125, + "margin_dpo/margin_std": 225.96490478515625, + "step": 368 + }, + { + "KL/chosen_KL_mean": -358.34112548828125, + "KL/mean": -433.00439453125, + "KL/rejected_KL_mean": -507.6676025390625, + "KL/std": 255.37953186035156, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0018820683471858501, + "fcm_dpo/delta": 0.12171748280525208, + "fcm_dpo/margin": 149.3264617919922, + "fcm_dpo/q_t": 0.43490248918533325, + "grad_norm": 28.77642059326172, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.4904497563838959, + "logits/rejected": -0.46972647309303284, + "logps/chosen": -429.72265625, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -598.9634399414062, + "loss": 1.1924, + "margin_dpo/margin_mean": 149.32644653320312, + "margin_dpo/margin_std": 282.73687744140625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -377.56915283203125, + "KL/mean": -451.96075439453125, + "KL/rejected_KL_mean": -526.352294921875, + "KL/std": 270.2765808105469, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0019240763504058123, + "fcm_dpo/delta": 0.11676926910877228, + "fcm_dpo/margin": 148.783203125, + "fcm_dpo/q_t": 0.4354085326194763, + "grad_norm": 36.105655670166016, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.506401777267456, + "logits/rejected": -0.49917715787887573, + "logps/chosen": -449.1766357421875, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -623.6121215820312, + "loss": 1.2, + "margin_dpo/margin_mean": 148.783203125, + "margin_dpo/margin_std": 300.480224609375, + "step": 370 + }, + { + "KL/chosen_KL_mean": -367.2420654296875, + "KL/mean": -475.67535400390625, + "KL/rejected_KL_mean": -584.108642578125, + "KL/std": 272.7081298828125, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0019323548767715693, + "fcm_dpo/delta": -0.020058294758200645, + "fcm_dpo/margin": 216.86659240722656, + "fcm_dpo/q_t": 0.40576615929603577, + "grad_norm": 26.2259521484375, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.5232386589050293, + "logits/rejected": -0.5076332092285156, + "logps/chosen": -436.65655517578125, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -683.2808837890625, + "loss": 1.0957, + "margin_dpo/margin_mean": 216.86659240722656, + "margin_dpo/margin_std": 312.64453125, + "step": 371 + }, + { + "KL/chosen_KL_mean": -358.1750183105469, + "KL/mean": -487.3031311035156, + "KL/rejected_KL_mean": -616.4312133789062, + "KL/std": 310.6046447753906, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.0018943841569125652, + "fcm_dpo/delta": -0.09401249885559082, + "fcm_dpo/margin": 258.2561950683594, + "fcm_dpo/q_t": 0.39255887269973755, + "grad_norm": 30.38518524169922, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.5213905572891235, + "logits/rejected": -0.5028508901596069, + "logps/chosen": -419.9930114746094, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -694.970703125, + "loss": 1.0531, + "margin_dpo/margin_mean": 258.2561950683594, + "margin_dpo/margin_std": 354.30523681640625, + "step": 372 + }, + { + "KL/chosen_KL_mean": -379.118896484375, + "KL/mean": -500.2996826171875, + "KL/rejected_KL_mean": -621.48046875, + "KL/std": 290.8080749511719, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.0018818657845258713, + "fcm_dpo/delta": -0.05905989930033684, + "fcm_dpo/margin": 242.36154174804688, + "fcm_dpo/q_t": 0.39455127716064453, + "grad_norm": 27.136159896850586, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.48000702261924744, + "logits/rejected": -0.4573609530925751, + "logps/chosen": -443.3360595703125, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -707.4400634765625, + "loss": 1.0518, + "margin_dpo/margin_mean": 242.36154174804688, + "margin_dpo/margin_std": 281.60528564453125, + "step": 373 + }, + { + "KL/chosen_KL_mean": -336.77154541015625, + "KL/mean": -444.8589782714844, + "KL/rejected_KL_mean": -552.9464111328125, + "KL/std": 312.1567687988281, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.001856320770457387, + "fcm_dpo/delta": -0.0018408633768558502, + "fcm_dpo/margin": 216.17483520507812, + "fcm_dpo/q_t": 0.4123731553554535, + "grad_norm": 29.0636043548584, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.5651968717575073, + "logits/rejected": -0.5458487272262573, + "logps/chosen": -407.4217529296875, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -646.5865478515625, + "loss": 1.1214, + "margin_dpo/margin_mean": 216.17481994628906, + "margin_dpo/margin_std": 352.5235595703125, + "step": 374 + }, + { + "KL/chosen_KL_mean": -333.574951171875, + "KL/mean": -441.3401794433594, + "KL/rejected_KL_mean": -549.1054077148438, + "KL/std": 246.93435668945312, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0018542553298175335, + "fcm_dpo/delta": -0.00033789873123168945, + "fcm_dpo/margin": 215.5304718017578, + "fcm_dpo/q_t": 0.40774768590927124, + "grad_norm": 31.13868522644043, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.5243902206420898, + "logits/rejected": -0.5246233940124512, + "logps/chosen": -393.6551513671875, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -638.043701171875, + "loss": 1.098, + "margin_dpo/margin_mean": 215.53048706054688, + "margin_dpo/margin_std": 292.2589111328125, + "step": 375 + }, + { + "KL/chosen_KL_mean": -320.7277526855469, + "KL/mean": -449.8273010253906, + "KL/rejected_KL_mean": -578.9268798828125, + "KL/std": 262.97601318359375, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0018427311442792416, + "fcm_dpo/delta": -0.07959697395563126, + "fcm_dpo/margin": 258.1990661621094, + "fcm_dpo/q_t": 0.3906724452972412, + "grad_norm": 21.562713623046875, + "learning_rate": 2.5e-07, + "logits/chosen": -0.4940425753593445, + "logits/rejected": -0.48502638936042786, + "logps/chosen": -383.3880615234375, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -684.4534912109375, + "loss": 1.0407, + "margin_dpo/margin_mean": 258.19903564453125, + "margin_dpo/margin_std": 307.8325500488281, + "step": 376 + }, + { + "KL/chosen_KL_mean": -309.9473571777344, + "KL/mean": -430.5098876953125, + "KL/rejected_KL_mean": -551.0723876953125, + "KL/std": 262.6895751953125, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0018268723506480455, + "fcm_dpo/delta": -0.04235388711094856, + "fcm_dpo/margin": 241.12506103515625, + "fcm_dpo/q_t": 0.3979926109313965, + "grad_norm": 28.506914138793945, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.434537410736084, + "logits/rejected": -0.4354628920555115, + "logps/chosen": -364.42608642578125, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -649.7757568359375, + "loss": 1.0542, + "margin_dpo/margin_mean": 241.12506103515625, + "margin_dpo/margin_std": 277.93939208984375, + "step": 377 + }, + { + "KL/chosen_KL_mean": -295.1936950683594, + "KL/mean": -424.91107177734375, + "KL/rejected_KL_mean": -554.6285400390625, + "KL/std": 245.08253479003906, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.0017997961258515716, + "fcm_dpo/delta": -0.07027687132358551, + "fcm_dpo/margin": 259.434814453125, + "fcm_dpo/q_t": 0.3899168372154236, + "grad_norm": 25.608421325683594, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.475490927696228, + "logits/rejected": -0.49409961700439453, + "logps/chosen": -340.2142333984375, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -642.675537109375, + "loss": 1.0248, + "margin_dpo/margin_mean": 259.434814453125, + "margin_dpo/margin_std": 260.9562072753906, + "step": 378 + }, + { + "KL/chosen_KL_mean": -335.8722839355469, + "KL/mean": -460.6631774902344, + "KL/rejected_KL_mean": -585.4541015625, + "KL/std": 250.936279296875, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0017757815076038241, + "fcm_dpo/delta": -0.04568084329366684, + "fcm_dpo/margin": 249.581787109375, + "fcm_dpo/q_t": 0.3978073000907898, + "grad_norm": 30.588621139526367, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.47063854336738586, + "logits/rejected": -0.49576616287231445, + "logps/chosen": -389.05438232421875, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -699.7542114257812, + "loss": 1.055, + "margin_dpo/margin_mean": 249.58177185058594, + "margin_dpo/margin_std": 292.3218078613281, + "step": 379 + }, + { + "KL/chosen_KL_mean": -328.56036376953125, + "KL/mean": -467.00640869140625, + "KL/rejected_KL_mean": -605.4524536132812, + "KL/std": 282.5246276855469, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0017567714676260948, + "fcm_dpo/delta": -0.09077489376068115, + "fcm_dpo/margin": 276.89208984375, + "fcm_dpo/q_t": 0.386593222618103, + "grad_norm": 31.498470306396484, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.45121973752975464, + "logits/rejected": -0.4780144691467285, + "logps/chosen": -379.91339111328125, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -709.6441650390625, + "loss": 1.0252, + "margin_dpo/margin_mean": 276.89208984375, + "margin_dpo/margin_std": 306.36444091796875, + "step": 380 + }, + { + "KL/chosen_KL_mean": -339.11285400390625, + "KL/mean": -437.67279052734375, + "KL/rejected_KL_mean": -536.2327880859375, + "KL/std": 237.19540405273438, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0017585060559213161, + "fcm_dpo/delta": 0.05531725659966469, + "fcm_dpo/margin": 197.11988830566406, + "fcm_dpo/q_t": 0.4206732213497162, + "grad_norm": 25.56667137145996, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4570884704589844, + "logits/rejected": -0.439144104719162, + "logps/chosen": -396.9158935546875, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -615.4521484375, + "loss": 1.1446, + "margin_dpo/margin_mean": 197.119873046875, + "margin_dpo/margin_std": 321.15301513671875, + "step": 381 + }, + { + "KL/chosen_KL_mean": -329.6747131347656, + "KL/mean": -452.9117431640625, + "KL/rejected_KL_mean": -576.1488037109375, + "KL/std": 232.52609252929688, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0017561506247147918, + "fcm_dpo/delta": -0.034348826855421066, + "fcm_dpo/margin": 246.47412109375, + "fcm_dpo/q_t": 0.39846402406692505, + "grad_norm": 32.8306884765625, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.509077787399292, + "logits/rejected": -0.5165588855743408, + "logps/chosen": -395.69500732421875, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -686.8590087890625, + "loss": 1.0461, + "margin_dpo/margin_mean": 246.47412109375, + "margin_dpo/margin_std": 253.2662353515625, + "step": 382 + }, + { + "KL/chosen_KL_mean": -342.23028564453125, + "KL/mean": -465.522705078125, + "KL/rejected_KL_mean": -588.815185546875, + "KL/std": 262.4213562011719, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0017451886087656021, + "fcm_dpo/delta": -0.03170797973871231, + "fcm_dpo/margin": 246.58486938476562, + "fcm_dpo/q_t": 0.4010571241378784, + "grad_norm": 33.12326431274414, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.44466179609298706, + "logits/rejected": -0.4535874128341675, + "logps/chosen": -392.62176513671875, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -682.531005859375, + "loss": 1.0836, + "margin_dpo/margin_mean": 246.58486938476562, + "margin_dpo/margin_std": 340.9044189453125, + "step": 383 + }, + { + "KL/chosen_KL_mean": -357.75701904296875, + "KL/mean": -453.1382751464844, + "KL/rejected_KL_mean": -548.51953125, + "KL/std": 238.6768341064453, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.001766695873811841, + "fcm_dpo/delta": 0.06458175182342529, + "fcm_dpo/margin": 190.7625274658203, + "fcm_dpo/q_t": 0.4200253486633301, + "grad_norm": 21.88326072692871, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.48681867122650146, + "logits/rejected": -0.4805874526500702, + "logps/chosen": -409.8031005859375, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -634.2804565429688, + "loss": 1.1145, + "margin_dpo/margin_mean": 190.76254272460938, + "margin_dpo/margin_std": 218.62911987304688, + "step": 384 + }, + { + "KL/chosen_KL_mean": -335.08123779296875, + "KL/mean": -452.1759948730469, + "KL/rejected_KL_mean": -569.270751953125, + "KL/std": 211.10403442382812, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0017625440377742052, + "fcm_dpo/delta": -0.013317081145942211, + "fcm_dpo/margin": 234.1895294189453, + "fcm_dpo/q_t": 0.4020652174949646, + "grad_norm": 32.658111572265625, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.4951225519180298, + "logits/rejected": -0.4684150815010071, + "logps/chosen": -400.6334228515625, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -647.0986938476562, + "loss": 1.0558, + "margin_dpo/margin_mean": 234.18951416015625, + "margin_dpo/margin_std": 232.9949188232422, + "step": 385 + }, + { + "KL/chosen_KL_mean": -356.7154541015625, + "KL/mean": -486.1207275390625, + "KL/rejected_KL_mean": -615.5260009765625, + "KL/std": 273.87554931640625, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0017484361305832863, + "fcm_dpo/delta": -0.05496610701084137, + "fcm_dpo/margin": 258.810546875, + "fcm_dpo/q_t": 0.3961135447025299, + "grad_norm": 23.645395278930664, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.47803959250450134, + "logits/rejected": -0.47327008843421936, + "logps/chosen": -414.93731689453125, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -707.8533935546875, + "loss": 1.055, + "margin_dpo/margin_mean": 258.810546875, + "margin_dpo/margin_std": 317.30010986328125, + "step": 386 + }, + { + "KL/chosen_KL_mean": -383.32470703125, + "KL/mean": -487.42926025390625, + "KL/rejected_KL_mean": -591.5338134765625, + "KL/std": 252.27149963378906, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.001744621666148305, + "fcm_dpo/delta": 0.03798586130142212, + "fcm_dpo/margin": 208.2091064453125, + "fcm_dpo/q_t": 0.4148293137550354, + "grad_norm": 29.001224517822266, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.5225445032119751, + "logits/rejected": -0.5023460388183594, + "logps/chosen": -449.7441711425781, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -683.7030029296875, + "loss": 1.1047, + "margin_dpo/margin_mean": 208.2091064453125, + "margin_dpo/margin_std": 255.5272216796875, + "step": 387 + }, + { + "KL/chosen_KL_mean": -354.7261962890625, + "KL/mean": -496.9158935546875, + "KL/rejected_KL_mean": -639.1056518554688, + "KL/std": 290.834228515625, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0017255207058042288, + "fcm_dpo/delta": -0.09575385600328445, + "fcm_dpo/margin": 284.37939453125, + "fcm_dpo/q_t": 0.38899484276771545, + "grad_norm": 23.755598068237305, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.46578872203826904, + "logits/rejected": -0.4780058264732361, + "logps/chosen": -404.85565185546875, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -743.5386962890625, + "loss": 1.0307, + "margin_dpo/margin_mean": 284.3794250488281, + "margin_dpo/margin_std": 337.2706298828125, + "step": 388 + }, + { + "KL/chosen_KL_mean": -389.77545166015625, + "KL/mean": -505.8580322265625, + "KL/rejected_KL_mean": -621.9406127929688, + "KL/std": 281.48638916015625, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.001719092484563589, + "fcm_dpo/delta": 0.0008805897086858749, + "fcm_dpo/margin": 232.16519165039062, + "fcm_dpo/q_t": 0.407645583152771, + "grad_norm": 29.629207611083984, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.5295721292495728, + "logits/rejected": -0.5217357873916626, + "logps/chosen": -447.6820068359375, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -699.8551635742188, + "loss": 1.0912, + "margin_dpo/margin_mean": 232.16519165039062, + "margin_dpo/margin_std": 305.3445129394531, + "step": 389 + }, + { + "KL/chosen_KL_mean": -364.65234375, + "KL/mean": -482.767333984375, + "KL/rejected_KL_mean": -600.88232421875, + "KL/std": 274.46929931640625, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.00171397696249187, + "fcm_dpo/delta": -0.005398150533437729, + "fcm_dpo/margin": 236.23001098632812, + "fcm_dpo/q_t": 0.4100305438041687, + "grad_norm": 27.02170181274414, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.4661334156990051, + "logits/rejected": -0.45682328939437866, + "logps/chosen": -413.87823486328125, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -686.4105224609375, + "loss": 1.1037, + "margin_dpo/margin_mean": 236.23001098632812, + "margin_dpo/margin_std": 352.0171203613281, + "step": 390 + }, + { + "KL/chosen_KL_mean": -361.19842529296875, + "KL/mean": -433.7296447753906, + "KL/rejected_KL_mean": -506.26092529296875, + "KL/std": 255.76431274414062, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0017586734611541033, + "fcm_dpo/delta": 0.14866049587726593, + "fcm_dpo/margin": 145.0625, + "fcm_dpo/q_t": 0.44111597537994385, + "grad_norm": 38.00566482543945, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.5272815227508545, + "logits/rejected": -0.5189157128334045, + "logps/chosen": -425.528076171875, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -592.9991455078125, + "loss": 1.2011, + "margin_dpo/margin_mean": 145.0625, + "margin_dpo/margin_std": 265.6080627441406, + "step": 391 + }, + { + "KL/chosen_KL_mean": -307.17938232421875, + "KL/mean": -442.08966064453125, + "KL/rejected_KL_mean": -576.9999389648438, + "KL/std": 263.717529296875, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0017566069727763534, + "fcm_dpo/delta": -0.07759811729192734, + "fcm_dpo/margin": 269.820556640625, + "fcm_dpo/q_t": 0.38913753628730774, + "grad_norm": 26.105369567871094, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.4802924394607544, + "logits/rejected": -0.47493991255760193, + "logps/chosen": -360.683349609375, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -679.3457641601562, + "loss": 1.0148, + "margin_dpo/margin_mean": 269.820556640625, + "margin_dpo/margin_std": 254.5288543701172, + "step": 392 + }, + { + "KL/chosen_KL_mean": -298.51666259765625, + "KL/mean": -408.068115234375, + "KL/rejected_KL_mean": -517.61962890625, + "KL/std": 264.71429443359375, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0017463632393628359, + "fcm_dpo/delta": 0.018034297972917557, + "fcm_dpo/margin": 219.1029510498047, + "fcm_dpo/q_t": 0.414645791053772, + "grad_norm": 22.201059341430664, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.47289371490478516, + "logits/rejected": -0.47130799293518066, + "logps/chosen": -344.9906005859375, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -589.5885009765625, + "loss": 1.1146, + "margin_dpo/margin_mean": 219.10296630859375, + "margin_dpo/margin_std": 332.18524169921875, + "step": 393 + }, + { + "KL/chosen_KL_mean": -332.33935546875, + "KL/mean": -446.65289306640625, + "KL/rejected_KL_mean": -560.9664306640625, + "KL/std": 277.8155212402344, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.0017484340351074934, + "fcm_dpo/delta": 0.0002336958423256874, + "fcm_dpo/margin": 228.62710571289062, + "fcm_dpo/q_t": 0.40847307443618774, + "grad_norm": 22.033977508544922, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.47436898946762085, + "logits/rejected": -0.473477840423584, + "logps/chosen": -385.2508850097656, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -651.7890625, + "loss": 1.0865, + "margin_dpo/margin_mean": 228.62710571289062, + "margin_dpo/margin_std": 292.49322509765625, + "step": 394 + }, + { + "KL/chosen_KL_mean": -329.78057861328125, + "KL/mean": -450.36175537109375, + "KL/rejected_KL_mean": -570.9429931640625, + "KL/std": 271.3304443359375, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0017403799574822187, + "fcm_dpo/delta": -0.02080959640443325, + "fcm_dpo/margin": 241.16241455078125, + "fcm_dpo/q_t": 0.4038548469543457, + "grad_norm": 28.20953941345215, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.5319548845291138, + "logits/rejected": -0.5265194177627563, + "logps/chosen": -392.32666015625, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -654.7255859375, + "loss": 1.0782, + "margin_dpo/margin_mean": 241.1623992919922, + "margin_dpo/margin_std": 310.57696533203125, + "step": 395 + }, + { + "KL/chosen_KL_mean": -343.78643798828125, + "KL/mean": -453.4306335449219, + "KL/rejected_KL_mean": -563.0748291015625, + "KL/std": 267.2041320800781, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.00174568104557693, + "fcm_dpo/delta": 0.01785963401198387, + "fcm_dpo/margin": 219.28836059570312, + "fcm_dpo/q_t": 0.4103149473667145, + "grad_norm": 22.445358276367188, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.5086914300918579, + "logits/rejected": -0.49094802141189575, + "logps/chosen": -412.78240966796875, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -651.721435546875, + "loss": 1.0822, + "margin_dpo/margin_mean": 219.28836059570312, + "margin_dpo/margin_std": 244.86111450195312, + "step": 396 + }, + { + "KL/chosen_KL_mean": -324.86578369140625, + "KL/mean": -463.35546875, + "KL/rejected_KL_mean": -601.84521484375, + "KL/std": 266.4302978515625, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.001722394721582532, + "fcm_dpo/delta": -0.08150285482406616, + "fcm_dpo/margin": 276.9794006347656, + "fcm_dpo/q_t": 0.3886115550994873, + "grad_norm": 36.389671325683594, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.5031261444091797, + "logits/rejected": -0.5105383396148682, + "logps/chosen": -386.1429443359375, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -704.9613037109375, + "loss": 1.0185, + "margin_dpo/margin_mean": 276.9794006347656, + "margin_dpo/margin_std": 277.23126220703125, + "step": 397 + }, + { + "KL/chosen_KL_mean": -358.7999267578125, + "KL/mean": -486.3828430175781, + "KL/rejected_KL_mean": -613.9657592773438, + "KL/std": 283.18798828125, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0017112932400777936, + "fcm_dpo/delta": -0.038348376750946045, + "fcm_dpo/margin": 255.16580200195312, + "fcm_dpo/q_t": 0.4006795883178711, + "grad_norm": 23.35751724243164, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.5430258512496948, + "logits/rejected": -0.5390101671218872, + "logps/chosen": -426.95147705078125, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -722.4893798828125, + "loss": 1.0657, + "margin_dpo/margin_mean": 255.16578674316406, + "margin_dpo/margin_std": 323.1393127441406, + "step": 398 + }, + { + "KL/chosen_KL_mean": -317.32110595703125, + "KL/mean": -435.22442626953125, + "KL/rejected_KL_mean": -553.127685546875, + "KL/std": 253.32781982421875, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0017030900344252586, + "fcm_dpo/delta": -0.0018346961587667465, + "fcm_dpo/margin": 235.8065948486328, + "fcm_dpo/q_t": 0.4074350595474243, + "grad_norm": 28.525602340698242, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.5288404226303101, + "logits/rejected": -0.4914571940898895, + "logps/chosen": -378.2109069824219, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -631.09326171875, + "loss": 1.091, + "margin_dpo/margin_mean": 235.8065948486328, + "margin_dpo/margin_std": 311.6751708984375, + "step": 399 + }, + { + "KL/chosen_KL_mean": -310.41033935546875, + "KL/mean": -476.3387451171875, + "KL/rejected_KL_mean": -642.2671508789062, + "KL/std": 266.9754638671875, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0016638061497360468, + "fcm_dpo/delta": -0.16146430373191833, + "fcm_dpo/margin": 331.8568115234375, + "fcm_dpo/q_t": 0.37014493346214294, + "grad_norm": 24.833024978637695, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.5226554274559021, + "logits/rejected": -0.5213466882705688, + "logps/chosen": -374.053955078125, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -747.5198364257812, + "loss": 0.9637, + "margin_dpo/margin_mean": 331.8568115234375, + "margin_dpo/margin_std": 293.438720703125, + "step": 400 + }, + { + "KL/chosen_KL_mean": -367.61737060546875, + "KL/mean": -459.95599365234375, + "KL/rejected_KL_mean": -552.2945556640625, + "KL/std": 282.56536865234375, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.001671886071562767, + "fcm_dpo/delta": 0.0942469984292984, + "fcm_dpo/margin": 184.67718505859375, + "fcm_dpo/q_t": 0.43156173825263977, + "grad_norm": 26.294349670410156, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.4922007918357849, + "logits/rejected": -0.48331043124198914, + "logps/chosen": -424.7804260253906, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -636.0870361328125, + "loss": 1.1949, + "margin_dpo/margin_mean": 184.6772003173828, + "margin_dpo/margin_std": 375.33209228515625, + "step": 401 + }, + { + "KL/chosen_KL_mean": -280.1576843261719, + "KL/mean": -454.3927307128906, + "KL/rejected_KL_mean": -628.6277465820312, + "KL/std": 302.77960205078125, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0016407633665949106, + "fcm_dpo/delta": -0.1820974349975586, + "fcm_dpo/margin": 348.4701232910156, + "fcm_dpo/q_t": 0.36712318658828735, + "grad_norm": 28.117359161376953, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.5108712315559387, + "logits/rejected": -0.5017030239105225, + "logps/chosen": -330.8980712890625, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -709.673828125, + "loss": 0.9499, + "margin_dpo/margin_mean": 348.4701232910156, + "margin_dpo/margin_std": 304.89813232421875, + "step": 402 + }, + { + "KL/chosen_KL_mean": -323.0272216796875, + "KL/mean": -456.74005126953125, + "KL/rejected_KL_mean": -590.452880859375, + "KL/std": 291.66162109375, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0016160355880856514, + "fcm_dpo/delta": -0.033635590225458145, + "fcm_dpo/margin": 267.42572021484375, + "fcm_dpo/q_t": 0.39999282360076904, + "grad_norm": 28.584901809692383, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.5131726264953613, + "logits/rejected": -0.5143396854400635, + "logps/chosen": -370.17449951171875, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -667.7195434570312, + "loss": 1.0572, + "margin_dpo/margin_mean": 267.42572021484375, + "margin_dpo/margin_std": 302.7037658691406, + "step": 403 + }, + { + "KL/chosen_KL_mean": -358.3605651855469, + "KL/mean": -487.010009765625, + "KL/rejected_KL_mean": -615.659423828125, + "KL/std": 278.3720703125, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0016080401837825775, + "fcm_dpo/delta": -0.01440177857875824, + "fcm_dpo/margin": 257.29888916015625, + "fcm_dpo/q_t": 0.4054141640663147, + "grad_norm": 31.57237434387207, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.5295671820640564, + "logits/rejected": -0.521980881690979, + "logps/chosen": -406.23583984375, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -692.814453125, + "loss": 1.0942, + "margin_dpo/margin_mean": 257.29888916015625, + "margin_dpo/margin_std": 360.68695068359375, + "step": 404 + }, + { + "KL/chosen_KL_mean": -400.57421875, + "KL/mean": -510.0733947753906, + "KL/rejected_KL_mean": -619.572509765625, + "KL/std": 305.08990478515625, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.0016325011383742094, + "fcm_dpo/delta": 0.042388975620269775, + "fcm_dpo/margin": 218.99832153320312, + "fcm_dpo/q_t": 0.41996970772743225, + "grad_norm": 36.47658157348633, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.5198633074760437, + "logits/rejected": -0.5131893157958984, + "logps/chosen": -465.73712158203125, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -706.7593383789062, + "loss": 1.1522, + "margin_dpo/margin_mean": 218.99830627441406, + "margin_dpo/margin_std": 371.7571105957031, + "step": 405 + }, + { + "KL/chosen_KL_mean": -373.141845703125, + "KL/mean": -518.8602905273438, + "KL/rejected_KL_mean": -664.5787353515625, + "KL/std": 316.84564208984375, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0016123302048072219, + "fcm_dpo/delta": -0.07341163605451584, + "fcm_dpo/margin": 291.43695068359375, + "fcm_dpo/q_t": 0.39349114894866943, + "grad_norm": 30.422624588012695, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.5416710376739502, + "logits/rejected": -0.5567722320556641, + "logps/chosen": -422.8826599121094, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -756.6574096679688, + "loss": 1.0455, + "margin_dpo/margin_mean": 291.4369201660156, + "margin_dpo/margin_std": 352.5291748046875, + "step": 406 + }, + { + "KL/chosen_KL_mean": -392.7001953125, + "KL/mean": -478.5469970703125, + "KL/rejected_KL_mean": -564.3937377929688, + "KL/std": 244.32614135742188, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0016374328406527638, + "fcm_dpo/delta": 0.12099770456552505, + "fcm_dpo/margin": 171.69357299804688, + "fcm_dpo/q_t": 0.43447309732437134, + "grad_norm": 34.30635452270508, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.5458344221115112, + "logits/rejected": -0.5158517360687256, + "logps/chosen": -449.0308837890625, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -641.9058837890625, + "loss": 1.1961, + "margin_dpo/margin_mean": 171.69357299804688, + "margin_dpo/margin_std": 327.2861328125, + "step": 407 + }, + { + "KL/chosen_KL_mean": -389.81781005859375, + "KL/mean": -493.51953125, + "KL/rejected_KL_mean": -597.2213134765625, + "KL/std": 235.57968139648438, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.001653898274526, + "fcm_dpo/delta": 0.0589529350399971, + "fcm_dpo/margin": 207.40350341796875, + "fcm_dpo/q_t": 0.41926515102386475, + "grad_norm": 28.1993350982666, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.573693037033081, + "logits/rejected": -0.5522395372390747, + "logps/chosen": -459.60711669921875, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -687.3182373046875, + "loss": 1.1173, + "margin_dpo/margin_mean": 207.40350341796875, + "margin_dpo/margin_std": 260.7626037597656, + "step": 408 + }, + { + "KL/chosen_KL_mean": -371.58984375, + "KL/mean": -472.3695068359375, + "KL/rejected_KL_mean": -573.1492919921875, + "KL/std": 261.2839050292969, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0016783499158918858, + "fcm_dpo/delta": 0.06352691352367401, + "fcm_dpo/margin": 201.55941772460938, + "fcm_dpo/q_t": 0.4214837849140167, + "grad_norm": 34.07803726196289, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.5670665502548218, + "logits/rejected": -0.5548566579818726, + "logps/chosen": -438.90728759765625, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -658.0535278320312, + "loss": 1.1303, + "margin_dpo/margin_mean": 201.55941772460938, + "margin_dpo/margin_std": 282.4700622558594, + "step": 409 + }, + { + "KL/chosen_KL_mean": -353.9989013671875, + "KL/mean": -472.0521240234375, + "KL/rejected_KL_mean": -590.1053466796875, + "KL/std": 255.26121520996094, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0016779222059994936, + "fcm_dpo/delta": 0.0039640069007873535, + "fcm_dpo/margin": 236.1064453125, + "fcm_dpo/q_t": 0.4067569971084595, + "grad_norm": 28.882566452026367, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.5682647228240967, + "logits/rejected": -0.5725410580635071, + "logps/chosen": -405.4642639160156, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -673.3043212890625, + "loss": 1.0835, + "margin_dpo/margin_mean": 236.1064453125, + "margin_dpo/margin_std": 285.48974609375, + "step": 410 + }, + { + "KL/chosen_KL_mean": -379.1360168457031, + "KL/mean": -494.1120300292969, + "KL/rejected_KL_mean": -609.0880737304688, + "KL/std": 283.6739501953125, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0016700525302439928, + "fcm_dpo/delta": 0.015278995037078857, + "fcm_dpo/margin": 229.95205688476562, + "fcm_dpo/q_t": 0.41327282786369324, + "grad_norm": 36.2697639465332, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.5964963436126709, + "logits/rejected": -0.6090751886367798, + "logps/chosen": -431.44329833984375, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -689.7830200195312, + "loss": 1.1135, + "margin_dpo/margin_mean": 229.95205688476562, + "margin_dpo/margin_std": 326.5926513671875, + "step": 411 + }, + { + "KL/chosen_KL_mean": -371.9578552246094, + "KL/mean": -494.80596923828125, + "KL/rejected_KL_mean": -617.654052734375, + "KL/std": 272.0191650390625, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0016836973372846842, + "fcm_dpo/delta": -0.014276554808020592, + "fcm_dpo/margin": 245.6961669921875, + "fcm_dpo/q_t": 0.4044150710105896, + "grad_norm": 37.83639907836914, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.6487008333206177, + "logits/rejected": -0.6855199933052063, + "logps/chosen": -425.10198974609375, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -717.71484375, + "loss": 1.0808, + "margin_dpo/margin_mean": 245.6961669921875, + "margin_dpo/margin_std": 314.57440185546875, + "step": 412 + }, + { + "KL/chosen_KL_mean": -409.2244873046875, + "KL/mean": -531.232666015625, + "KL/rejected_KL_mean": -653.2408447265625, + "KL/std": 287.9561462402344, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0016818228177726269, + "fcm_dpo/delta": -0.01092137023806572, + "fcm_dpo/margin": 244.016357421875, + "fcm_dpo/q_t": 0.4036375880241394, + "grad_norm": 36.895450592041016, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.5873122811317444, + "logits/rejected": -0.5993106365203857, + "logps/chosen": -470.80645751953125, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -752.7142333984375, + "loss": 1.0823, + "margin_dpo/margin_mean": 244.016357421875, + "margin_dpo/margin_std": 310.86529541015625, + "step": 413 + }, + { + "KL/chosen_KL_mean": -375.28753662109375, + "KL/mean": -503.64361572265625, + "KL/rejected_KL_mean": -631.999755859375, + "KL/std": 261.2779846191406, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0016790458466857672, + "fcm_dpo/delta": -0.032990626990795135, + "fcm_dpo/margin": 256.7121276855469, + "fcm_dpo/q_t": 0.3996039032936096, + "grad_norm": 36.35009002685547, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.54841548204422, + "logits/rejected": -0.545833170413971, + "logps/chosen": -421.9190368652344, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -719.646240234375, + "loss": 1.0625, + "margin_dpo/margin_mean": 256.7121276855469, + "margin_dpo/margin_std": 296.39483642578125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -398.240234375, + "KL/mean": -497.1719970703125, + "KL/rejected_KL_mean": -596.103759765625, + "KL/std": 269.16583251953125, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.001677217660471797, + "fcm_dpo/delta": 0.07045261561870575, + "fcm_dpo/margin": 197.86349487304688, + "fcm_dpo/q_t": 0.4225271940231323, + "grad_norm": 30.878511428833008, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.6334292888641357, + "logits/rejected": -0.6316261291503906, + "logps/chosen": -476.8585205078125, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -696.581298828125, + "loss": 1.1411, + "margin_dpo/margin_mean": 197.86349487304688, + "margin_dpo/margin_std": 295.06793212890625, + "step": 415 + }, + { + "KL/chosen_KL_mean": -373.45452880859375, + "KL/mean": -528.5955810546875, + "KL/rejected_KL_mean": -683.736572265625, + "KL/std": 301.2171630859375, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.001664166571572423, + "fcm_dpo/delta": -0.1225137934088707, + "fcm_dpo/margin": 310.28204345703125, + "fcm_dpo/q_t": 0.37982699275016785, + "grad_norm": 43.10813903808594, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.6585408449172974, + "logits/rejected": -0.6719676852226257, + "logps/chosen": -431.733642578125, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -774.3052978515625, + "loss": 0.9924, + "margin_dpo/margin_mean": 310.28204345703125, + "margin_dpo/margin_std": 296.31573486328125, + "step": 416 + }, + { + "KL/chosen_KL_mean": -372.3450012207031, + "KL/mean": -507.4084777832031, + "KL/rejected_KL_mean": -642.4720458984375, + "KL/std": 284.5958251953125, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.0016403777990490198, + "fcm_dpo/delta": -0.045090239495038986, + "fcm_dpo/margin": 270.1269836425781, + "fcm_dpo/q_t": 0.39596399664878845, + "grad_norm": 34.500221252441406, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.6143770217895508, + "logits/rejected": -0.5956833362579346, + "logps/chosen": -422.543701171875, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -710.6238403320312, + "loss": 1.0399, + "margin_dpo/margin_mean": 270.12701416015625, + "margin_dpo/margin_std": 276.19189453125, + "step": 417 + }, + { + "KL/chosen_KL_mean": -409.942626953125, + "KL/mean": -547.230712890625, + "KL/rejected_KL_mean": -684.5186767578125, + "KL/std": 322.7239990234375, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0016350080259144306, + "fcm_dpo/delta": -0.05223679542541504, + "fcm_dpo/margin": 274.5760192871094, + "fcm_dpo/q_t": 0.39974966645240784, + "grad_norm": 29.770610809326172, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.6951717138290405, + "logits/rejected": -0.6750404238700867, + "logps/chosen": -491.9211120605469, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -789.210205078125, + "loss": 1.0767, + "margin_dpo/margin_mean": 274.5760192871094, + "margin_dpo/margin_std": 369.99847412109375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -369.484375, + "KL/mean": -522.38818359375, + "KL/rejected_KL_mean": -675.2919921875, + "KL/std": 289.92828369140625, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0015955464914441109, + "fcm_dpo/delta": -0.09240511804819107, + "fcm_dpo/margin": 305.8075866699219, + "fcm_dpo/q_t": 0.3869694471359253, + "grad_norm": 29.81219482421875, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.6173703670501709, + "logits/rejected": -0.6172693371772766, + "logps/chosen": -422.43304443359375, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -766.8750610351562, + "loss": 1.0245, + "margin_dpo/margin_mean": 305.8076171875, + "margin_dpo/margin_std": 338.61688232421875, + "step": 419 + }, + { + "KL/chosen_KL_mean": -466.75408935546875, + "KL/mean": -559.7321166992188, + "KL/rejected_KL_mean": -652.7100830078125, + "KL/std": 303.958740234375, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0016136702615767717, + "fcm_dpo/delta": 0.10288095474243164, + "fcm_dpo/margin": 185.9560089111328, + "fcm_dpo/q_t": 0.4333432912826538, + "grad_norm": 63.88848876953125, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.6588238477706909, + "logits/rejected": -0.621021032333374, + "logps/chosen": -544.5240478515625, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -722.0299682617188, + "loss": 1.207, + "margin_dpo/margin_mean": 185.95602416992188, + "margin_dpo/margin_std": 404.29864501953125, + "step": 420 + }, + { + "KL/chosen_KL_mean": -386.9909362792969, + "KL/mean": -491.9422302246094, + "KL/rejected_KL_mean": -596.8934326171875, + "KL/std": 296.73309326171875, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.001641649752855301, + "fcm_dpo/delta": 0.05669290944933891, + "fcm_dpo/margin": 209.902587890625, + "fcm_dpo/q_t": 0.41849058866500854, + "grad_norm": 27.782373428344727, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.686546802520752, + "logits/rejected": -0.672784686088562, + "logps/chosen": -440.7568054199219, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -686.1749267578125, + "loss": 1.135, + "margin_dpo/margin_mean": 209.902587890625, + "margin_dpo/margin_std": 305.8330078125, + "step": 421 + }, + { + "KL/chosen_KL_mean": -433.81988525390625, + "KL/mean": -554.65283203125, + "KL/rejected_KL_mean": -675.48583984375, + "KL/std": 304.8125, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0016393410041928291, + "fcm_dpo/delta": 0.003984665498137474, + "fcm_dpo/margin": 241.66592407226562, + "fcm_dpo/q_t": 0.4081314206123352, + "grad_norm": 31.474258422851562, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.6679846048355103, + "logits/rejected": -0.6701922416687012, + "logps/chosen": -502.45367431640625, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -763.349365234375, + "loss": 1.1075, + "margin_dpo/margin_mean": 241.6659393310547, + "margin_dpo/margin_std": 354.5689697265625, + "step": 422 + }, + { + "KL/chosen_KL_mean": -401.0167236328125, + "KL/mean": -538.4049072265625, + "KL/rejected_KL_mean": -675.793212890625, + "KL/std": 285.955078125, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0016276114620268345, + "fcm_dpo/delta": -0.049456000328063965, + "fcm_dpo/margin": 274.7764892578125, + "fcm_dpo/q_t": 0.3961997926235199, + "grad_norm": 35.502323150634766, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.6570872664451599, + "logits/rejected": -0.6668051481246948, + "logps/chosen": -456.00701904296875, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -762.0997314453125, + "loss": 1.047, + "margin_dpo/margin_mean": 274.7764892578125, + "margin_dpo/margin_std": 309.498046875, + "step": 423 + }, + { + "KL/chosen_KL_mean": -369.1432800292969, + "KL/mean": -490.0072021484375, + "KL/rejected_KL_mean": -610.87109375, + "KL/std": 279.67279052734375, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0016186567954719067, + "fcm_dpo/delta": 0.008693736046552658, + "fcm_dpo/margin": 241.72776794433594, + "fcm_dpo/q_t": 0.4110856354236603, + "grad_norm": 47.914939880371094, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.6212701797485352, + "logits/rejected": -0.5964733362197876, + "logps/chosen": -425.15521240234375, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -677.3500366210938, + "loss": 1.1031, + "margin_dpo/margin_mean": 241.727783203125, + "margin_dpo/margin_std": 337.23553466796875, + "step": 424 + }, + { + "KL/chosen_KL_mean": -412.35693359375, + "KL/mean": -504.414306640625, + "KL/rejected_KL_mean": -596.4716796875, + "KL/std": 263.9962158203125, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.001654601190239191, + "fcm_dpo/delta": 0.09802168607711792, + "fcm_dpo/margin": 184.11480712890625, + "fcm_dpo/q_t": 0.42791393399238586, + "grad_norm": 46.217079162597656, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.6049522757530212, + "logits/rejected": -0.6060476303100586, + "logps/chosen": -459.22589111328125, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -692.397216796875, + "loss": 1.1748, + "margin_dpo/margin_mean": 184.1148223876953, + "margin_dpo/margin_std": 326.7667236328125, + "step": 425 + }, + { + "KL/chosen_KL_mean": -371.05889892578125, + "KL/mean": -494.7485046386719, + "KL/rejected_KL_mean": -618.4381103515625, + "KL/std": 270.75262451171875, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0016634220955893397, + "fcm_dpo/delta": -0.012346841394901276, + "fcm_dpo/margin": 247.37916564941406, + "fcm_dpo/q_t": 0.40408235788345337, + "grad_norm": 30.221540451049805, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.6098922491073608, + "logits/rejected": -0.5774872303009033, + "logps/chosen": -447.6424560546875, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -699.7047119140625, + "loss": 1.0765, + "margin_dpo/margin_mean": 247.37916564941406, + "margin_dpo/margin_std": 296.57476806640625, + "step": 426 + }, + { + "KL/chosen_KL_mean": -348.84161376953125, + "KL/mean": -439.01953125, + "KL/rejected_KL_mean": -529.197509765625, + "KL/std": 231.7161407470703, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0016764425672590733, + "fcm_dpo/delta": 0.10081073641777039, + "fcm_dpo/margin": 180.35585021972656, + "fcm_dpo/q_t": 0.43000850081443787, + "grad_norm": 26.96040916442871, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.6408891081809998, + "logits/rejected": -0.6237009763717651, + "logps/chosen": -413.69549560546875, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -607.7634887695312, + "loss": 1.1592, + "margin_dpo/margin_mean": 180.35585021972656, + "margin_dpo/margin_std": 278.6473388671875, + "step": 427 + }, + { + "KL/chosen_KL_mean": -433.99176025390625, + "KL/mean": -557.658203125, + "KL/rejected_KL_mean": -681.3245849609375, + "KL/std": 305.66961669921875, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.001678521279245615, + "fcm_dpo/delta": -0.01613050140440464, + "fcm_dpo/margin": 247.33279418945312, + "fcm_dpo/q_t": 0.40512609481811523, + "grad_norm": 40.6098518371582, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.6067614555358887, + "logits/rejected": -0.6123305559158325, + "logps/chosen": -496.6284484863281, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -784.6064453125, + "loss": 1.107, + "margin_dpo/margin_mean": 247.33279418945312, + "margin_dpo/margin_std": 375.6197509765625, + "step": 428 + }, + { + "KL/chosen_KL_mean": -428.03472900390625, + "KL/mean": -526.74609375, + "KL/rejected_KL_mean": -625.45751953125, + "KL/std": 275.07623291015625, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0017113164067268372, + "fcm_dpo/delta": 0.0628078281879425, + "fcm_dpo/margin": 197.42282104492188, + "fcm_dpo/q_t": 0.42306485772132874, + "grad_norm": 42.88612365722656, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.6413565874099731, + "logits/rejected": -0.626760721206665, + "logps/chosen": -509.26873779296875, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -717.25244140625, + "loss": 1.1636, + "margin_dpo/margin_mean": 197.42282104492188, + "margin_dpo/margin_std": 342.1004333496094, + "step": 429 + }, + { + "KL/chosen_KL_mean": -348.1343994140625, + "KL/mean": -448.15447998046875, + "KL/rejected_KL_mean": -548.174560546875, + "KL/std": 243.23687744140625, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0017218288267031312, + "fcm_dpo/delta": 0.057404372841119766, + "fcm_dpo/margin": 200.04013061523438, + "fcm_dpo/q_t": 0.42005741596221924, + "grad_norm": 31.768253326416016, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.5702426433563232, + "logits/rejected": -0.5682145357131958, + "logps/chosen": -409.05474853515625, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -652.5973510742188, + "loss": 1.122, + "margin_dpo/margin_mean": 200.04013061523438, + "margin_dpo/margin_std": 269.5069580078125, + "step": 430 + }, + { + "KL/chosen_KL_mean": -327.10601806640625, + "KL/mean": -474.724609375, + "KL/rejected_KL_mean": -622.34326171875, + "KL/std": 274.1042785644531, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0016950183780863881, + "fcm_dpo/delta": -0.10589279234409332, + "fcm_dpo/margin": 295.23724365234375, + "fcm_dpo/q_t": 0.3836705684661865, + "grad_norm": 27.5445556640625, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.5927987694740295, + "logits/rejected": -0.5998170971870422, + "logps/chosen": -384.45477294921875, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -715.1834716796875, + "loss": 1.0114, + "margin_dpo/margin_mean": 295.23724365234375, + "margin_dpo/margin_std": 304.465087890625, + "step": 431 + }, + { + "KL/chosen_KL_mean": -333.78057861328125, + "KL/mean": -475.11285400390625, + "KL/rejected_KL_mean": -616.4451293945312, + "KL/std": 270.1529541015625, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.001678573782555759, + "fcm_dpo/delta": -0.07845490425825119, + "fcm_dpo/margin": 282.66455078125, + "fcm_dpo/q_t": 0.38971948623657227, + "grad_norm": 31.19795799255371, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.561385452747345, + "logits/rejected": -0.5513102412223816, + "logps/chosen": -404.8553771972656, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -715.024658203125, + "loss": 1.0205, + "margin_dpo/margin_mean": 282.6645202636719, + "margin_dpo/margin_std": 275.1983642578125, + "step": 432 + }, + { + "KL/chosen_KL_mean": -429.03680419921875, + "KL/mean": -521.4520263671875, + "KL/rejected_KL_mean": -613.8671264648438, + "KL/std": 270.37274169921875, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0016868317034095526, + "fcm_dpo/delta": 0.09088847786188126, + "fcm_dpo/margin": 184.83030700683594, + "fcm_dpo/q_t": 0.42558449506759644, + "grad_norm": 40.01298522949219, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.6517413854598999, + "logits/rejected": -0.6408432126045227, + "logps/chosen": -487.30999755859375, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -709.8179931640625, + "loss": 1.1683, + "margin_dpo/margin_mean": 184.83029174804688, + "margin_dpo/margin_std": 319.0679016113281, + "step": 433 + }, + { + "KL/chosen_KL_mean": -376.2447509765625, + "KL/mean": -493.01629638671875, + "KL/rejected_KL_mean": -609.787841796875, + "KL/std": 280.75927734375, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0016879650065675378, + "fcm_dpo/delta": 0.005812322720885277, + "fcm_dpo/margin": 233.54312133789062, + "fcm_dpo/q_t": 0.41424840688705444, + "grad_norm": 25.016590118408203, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.6333714127540588, + "logits/rejected": -0.6362388134002686, + "logps/chosen": -438.21844482421875, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -688.2864990234375, + "loss": 1.1078, + "margin_dpo/margin_mean": 233.54312133789062, + "margin_dpo/margin_std": 356.24835205078125, + "step": 434 + }, + { + "KL/chosen_KL_mean": -342.070556640625, + "KL/mean": -470.48260498046875, + "KL/rejected_KL_mean": -598.8946533203125, + "KL/std": 271.7498779296875, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0016836894210428, + "fcm_dpo/delta": -0.033942632377147675, + "fcm_dpo/margin": 256.8241271972656, + "fcm_dpo/q_t": 0.3996688425540924, + "grad_norm": 23.19141960144043, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.6687694787979126, + "logits/rejected": -0.663030743598938, + "logps/chosen": -393.5726318359375, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -686.4615478515625, + "loss": 1.0642, + "margin_dpo/margin_mean": 256.8240966796875, + "margin_dpo/margin_std": 301.7302551269531, + "step": 435 + }, + { + "KL/chosen_KL_mean": -357.0111083984375, + "KL/mean": -466.84375, + "KL/rejected_KL_mean": -576.6763916015625, + "KL/std": 241.30087280273438, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0016856963047757745, + "fcm_dpo/delta": 0.030778564512729645, + "fcm_dpo/margin": 219.66525268554688, + "fcm_dpo/q_t": 0.4138760566711426, + "grad_norm": 37.03017807006836, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.6477811336517334, + "logits/rejected": -0.6223078370094299, + "logps/chosen": -428.4148254394531, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -659.4041748046875, + "loss": 1.1047, + "margin_dpo/margin_mean": 219.66525268554688, + "margin_dpo/margin_std": 284.75469970703125, + "step": 436 + }, + { + "KL/chosen_KL_mean": -357.8575439453125, + "KL/mean": -469.2908935546875, + "KL/rejected_KL_mean": -580.7242431640625, + "KL/std": 230.9532012939453, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0016966592520475388, + "fcm_dpo/delta": 0.022726206108927727, + "fcm_dpo/margin": 222.86676025390625, + "fcm_dpo/q_t": 0.41135555505752563, + "grad_norm": 27.248863220214844, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.6390538811683655, + "logits/rejected": -0.6193490028381348, + "logps/chosen": -422.601806640625, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -662.767822265625, + "loss": 1.0966, + "margin_dpo/margin_mean": 222.86676025390625, + "margin_dpo/margin_std": 277.84088134765625, + "step": 437 + }, + { + "KL/chosen_KL_mean": -362.8171691894531, + "KL/mean": -491.82318115234375, + "KL/rejected_KL_mean": -620.8292236328125, + "KL/std": 264.3558654785156, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0016846774378791451, + "fcm_dpo/delta": -0.036728233098983765, + "fcm_dpo/margin": 258.0119934082031, + "fcm_dpo/q_t": 0.3982524573802948, + "grad_norm": 35.91462707519531, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.6329758167266846, + "logits/rejected": -0.6176602840423584, + "logps/chosen": -421.8358459472656, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -703.906005859375, + "loss": 1.0525, + "margin_dpo/margin_mean": 258.011962890625, + "margin_dpo/margin_std": 281.8877258300781, + "step": 438 + }, + { + "KL/chosen_KL_mean": -375.19061279296875, + "KL/mean": -478.79888916015625, + "KL/rejected_KL_mean": -582.4071044921875, + "KL/std": 264.5539855957031, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0016852838452905416, + "fcm_dpo/delta": 0.05125600844621658, + "fcm_dpo/margin": 207.216552734375, + "fcm_dpo/q_t": 0.42029955983161926, + "grad_norm": 28.194957733154297, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.6310042142868042, + "logits/rejected": -0.6362867951393127, + "logps/chosen": -428.97467041015625, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -666.392578125, + "loss": 1.136, + "margin_dpo/margin_mean": 207.216552734375, + "margin_dpo/margin_std": 298.20703125, + "step": 439 + }, + { + "KL/chosen_KL_mean": -399.1997985839844, + "KL/mean": -517.9849853515625, + "KL/rejected_KL_mean": -636.7700805664062, + "KL/std": 325.75286865234375, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0017090074252337217, + "fcm_dpo/delta": -0.006412219256162643, + "fcm_dpo/margin": 237.5702667236328, + "fcm_dpo/q_t": 0.4094950556755066, + "grad_norm": 41.53138732910156, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.6875945329666138, + "logits/rejected": -0.689441978931427, + "logps/chosen": -477.76654052734375, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -733.267822265625, + "loss": 1.0934, + "margin_dpo/margin_mean": 237.57025146484375, + "margin_dpo/margin_std": 332.59210205078125, + "step": 440 + }, + { + "KL/chosen_KL_mean": -449.48992919921875, + "KL/mean": -570.3905639648438, + "KL/rejected_KL_mean": -691.2911987304688, + "KL/std": 332.9839782714844, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0017063781851902604, + "fcm_dpo/delta": -0.013367945328354836, + "fcm_dpo/margin": 241.80125427246094, + "fcm_dpo/q_t": 0.410023033618927, + "grad_norm": 44.22810745239258, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.6582868099212646, + "logits/rejected": -0.6489556431770325, + "logps/chosen": -510.3143615722656, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -787.761962890625, + "loss": 1.1257, + "margin_dpo/margin_mean": 241.80125427246094, + "margin_dpo/margin_std": 402.00970458984375, + "step": 441 + }, + { + "KL/chosen_KL_mean": -373.7994689941406, + "KL/mean": -514.6734619140625, + "KL/rejected_KL_mean": -655.5475463867188, + "KL/std": 310.6441650390625, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0016909808618947864, + "fcm_dpo/delta": -0.0807371512055397, + "fcm_dpo/margin": 281.7480773925781, + "fcm_dpo/q_t": 0.3929908275604248, + "grad_norm": 27.9097900390625, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.6640024185180664, + "logits/rejected": -0.6619168519973755, + "logps/chosen": -420.8106689453125, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -732.0867919921875, + "loss": 1.0467, + "margin_dpo/margin_mean": 281.7480773925781, + "margin_dpo/margin_std": 342.1705322265625, + "step": 442 + }, + { + "KL/chosen_KL_mean": -435.946533203125, + "KL/mean": -527.93115234375, + "KL/rejected_KL_mean": -619.915771484375, + "KL/std": 322.6178894042969, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.001686213887296617, + "fcm_dpo/delta": 0.09258534759283066, + "fcm_dpo/margin": 183.96923828125, + "fcm_dpo/q_t": 0.4317651391029358, + "grad_norm": 33.63398361206055, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.6458783149719238, + "logits/rejected": -0.6255953907966614, + "logps/chosen": -507.21954345703125, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -706.595703125, + "loss": 1.2015, + "margin_dpo/margin_mean": 183.96923828125, + "margin_dpo/margin_std": 386.4306640625, + "step": 443 + }, + { + "KL/chosen_KL_mean": -420.94146728515625, + "KL/mean": -566.6063842773438, + "KL/rejected_KL_mean": -712.2713012695312, + "KL/std": 331.0177307128906, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.0016750325448811054, + "fcm_dpo/delta": -0.09291477501392365, + "fcm_dpo/margin": 291.329833984375, + "fcm_dpo/q_t": 0.3905254602432251, + "grad_norm": 32.249019622802734, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.5964144468307495, + "logits/rejected": -0.6076514720916748, + "logps/chosen": -478.1551818847656, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -809.5261840820312, + "loss": 1.0416, + "margin_dpo/margin_mean": 291.329833984375, + "margin_dpo/margin_std": 369.47515869140625, + "step": 444 + }, + { + "KL/chosen_KL_mean": -380.491943359375, + "KL/mean": -518.0828247070312, + "KL/rejected_KL_mean": -655.6736450195312, + "KL/std": 273.7188415527344, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0016589898150414228, + "fcm_dpo/delta": -0.05918622761964798, + "fcm_dpo/margin": 275.18170166015625, + "fcm_dpo/q_t": 0.39586469531059265, + "grad_norm": 29.30715560913086, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.6524978280067444, + "logits/rejected": -0.6507973670959473, + "logps/chosen": -447.791748046875, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -748.3563232421875, + "loss": 1.0511, + "margin_dpo/margin_mean": 275.18170166015625, + "margin_dpo/margin_std": 327.972900390625, + "step": 445 + }, + { + "KL/chosen_KL_mean": -354.2855224609375, + "KL/mean": -501.6607360839844, + "KL/rejected_KL_mean": -649.0359497070312, + "KL/std": 295.178466796875, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.0016413007397204638, + "fcm_dpo/delta": -0.08833765238523483, + "fcm_dpo/margin": 294.7503967285156, + "fcm_dpo/q_t": 0.387233704328537, + "grad_norm": 29.745594024658203, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.6606038808822632, + "logits/rejected": -0.6746002435684204, + "logps/chosen": -413.3840026855469, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -750.3001098632812, + "loss": 1.0205, + "margin_dpo/margin_mean": 294.75042724609375, + "margin_dpo/margin_std": 299.39874267578125, + "step": 446 + }, + { + "KL/chosen_KL_mean": -386.55181884765625, + "KL/mean": -541.65966796875, + "KL/rejected_KL_mean": -696.767578125, + "KL/std": 355.6910400390625, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.00159403495490551, + "fcm_dpo/delta": -0.0999402105808258, + "fcm_dpo/margin": 310.21575927734375, + "fcm_dpo/q_t": 0.39024585485458374, + "grad_norm": 33.842323303222656, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.7196981906890869, + "logits/rejected": -0.7375032305717468, + "logps/chosen": -442.62713623046875, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -795.46240234375, + "loss": 1.0379, + "margin_dpo/margin_mean": 310.2157897949219, + "margin_dpo/margin_std": 400.603515625, + "step": 447 + }, + { + "KL/chosen_KL_mean": -435.7290954589844, + "KL/mean": -586.072021484375, + "KL/rejected_KL_mean": -736.4149780273438, + "KL/std": 317.54486083984375, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0015779135283082724, + "fcm_dpo/delta": -0.07813362032175064, + "fcm_dpo/margin": 300.68585205078125, + "fcm_dpo/q_t": 0.3919687271118164, + "grad_norm": 41.811561584472656, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.6214036345481873, + "logits/rejected": -0.6313283443450928, + "logps/chosen": -495.7329406738281, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -838.6796264648438, + "loss": 1.0439, + "margin_dpo/margin_mean": 300.68585205078125, + "margin_dpo/margin_std": 364.5878601074219, + "step": 448 + }, + { + "KL/chosen_KL_mean": -453.650634765625, + "KL/mean": -605.6881103515625, + "KL/rejected_KL_mean": -757.7255249023438, + "KL/std": 384.36376953125, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.0015459386631846428, + "fcm_dpo/delta": -0.07378433644771576, + "fcm_dpo/margin": 304.07489013671875, + "fcm_dpo/q_t": 0.39799103140830994, + "grad_norm": 34.471744537353516, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.664047360420227, + "logits/rejected": -0.6631011962890625, + "logps/chosen": -521.17724609375, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -846.3224487304688, + "loss": 1.0789, + "margin_dpo/margin_mean": 304.07489013671875, + "margin_dpo/margin_std": 463.5087585449219, + "step": 449 + }, + { + "KL/chosen_KL_mean": -374.59564208984375, + "KL/mean": -544.0880126953125, + "KL/rejected_KL_mean": -713.5804443359375, + "KL/std": 343.8946533203125, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0015177726745605469, + "fcm_dpo/delta": -0.1205982118844986, + "fcm_dpo/margin": 338.9848327636719, + "fcm_dpo/q_t": 0.38020819425582886, + "grad_norm": 36.702392578125, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.720793604850769, + "logits/rejected": -0.7371557950973511, + "logps/chosen": -431.7037353515625, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -816.33544921875, + "loss": 0.9989, + "margin_dpo/margin_mean": 338.9848327636719, + "margin_dpo/margin_std": 342.6279296875, + "step": 450 + }, + { + "KL/chosen_KL_mean": -526.224853515625, + "KL/mean": -648.7066650390625, + "KL/rejected_KL_mean": -771.1884765625, + "KL/std": 415.2115478515625, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0015166854718700051, + "fcm_dpo/delta": 0.029389221221208572, + "fcm_dpo/margin": 244.9636688232422, + "fcm_dpo/q_t": 0.4122239053249359, + "grad_norm": 52.8525505065918, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.780580461025238, + "logits/rejected": -0.7578041553497314, + "logps/chosen": -584.6937255859375, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -844.117919921875, + "loss": 1.1722, + "margin_dpo/margin_mean": 244.9636688232422, + "margin_dpo/margin_std": 483.06982421875, + "step": 451 + }, + { + "KL/chosen_KL_mean": -373.4771423339844, + "KL/mean": -519.2084350585938, + "KL/rejected_KL_mean": -664.939697265625, + "KL/std": 316.89373779296875, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0015040370635688305, + "fcm_dpo/delta": -0.040243491530418396, + "fcm_dpo/margin": 291.4625549316406, + "fcm_dpo/q_t": 0.39707738161087036, + "grad_norm": 22.58098030090332, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.6820767521858215, + "logits/rejected": -0.7009351849555969, + "logps/chosen": -423.86767578125, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -762.7111206054688, + "loss": 1.0493, + "margin_dpo/margin_mean": 291.46258544921875, + "margin_dpo/margin_std": 317.8357849121094, + "step": 452 + }, + { + "KL/chosen_KL_mean": -431.75054931640625, + "KL/mean": -572.9964599609375, + "KL/rejected_KL_mean": -714.242431640625, + "KL/std": 321.07098388671875, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0014946234878152609, + "fcm_dpo/delta": -0.023362088948488235, + "fcm_dpo/margin": 282.49188232421875, + "fcm_dpo/q_t": 0.405646413564682, + "grad_norm": 37.91874313354492, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.7189410924911499, + "logits/rejected": -0.7047437429428101, + "logps/chosen": -489.46539306640625, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -796.4498291015625, + "loss": 1.0863, + "margin_dpo/margin_mean": 282.49188232421875, + "margin_dpo/margin_std": 395.4425048828125, + "step": 453 + }, + { + "KL/chosen_KL_mean": -525.0638427734375, + "KL/mean": -682.2659301757812, + "KL/rejected_KL_mean": -839.468017578125, + "KL/std": 373.33837890625, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0014773803995922208, + "fcm_dpo/delta": -0.06795372813940048, + "fcm_dpo/margin": 314.40423583984375, + "fcm_dpo/q_t": 0.3950144946575165, + "grad_norm": 35.617008209228516, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.7618668079376221, + "logits/rejected": -0.7530739307403564, + "logps/chosen": -586.0094604492188, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -924.4188232421875, + "loss": 1.0633, + "margin_dpo/margin_mean": 314.40423583984375, + "margin_dpo/margin_std": 428.0843505859375, + "step": 454 + }, + { + "KL/chosen_KL_mean": -433.1183166503906, + "KL/mean": -620.3408203125, + "KL/rejected_KL_mean": -807.5633544921875, + "KL/std": 376.65289306640625, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0014526566956192255, + "fcm_dpo/delta": -0.1520988792181015, + "fcm_dpo/margin": 374.445068359375, + "fcm_dpo/q_t": 0.37818747758865356, + "grad_norm": 32.71326446533203, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.7473499774932861, + "logits/rejected": -0.7715774774551392, + "logps/chosen": -478.0050048828125, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -922.8648681640625, + "loss": 1.0058, + "margin_dpo/margin_mean": 374.445068359375, + "margin_dpo/margin_std": 427.50140380859375, + "step": 455 + }, + { + "KL/chosen_KL_mean": -452.8604736328125, + "KL/mean": -624.6705322265625, + "KL/rejected_KL_mean": -796.4805908203125, + "KL/std": 383.74267578125, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0014142689760774374, + "fcm_dpo/delta": -0.09029080718755722, + "fcm_dpo/margin": 343.62017822265625, + "fcm_dpo/q_t": 0.38965851068496704, + "grad_norm": 29.38023567199707, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.720964789390564, + "logits/rejected": -0.7460182309150696, + "logps/chosen": -509.897216796875, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -901.698486328125, + "loss": 1.0212, + "margin_dpo/margin_mean": 343.62017822265625, + "margin_dpo/margin_std": 378.78033447265625, + "step": 456 + }, + { + "KL/chosen_KL_mean": -428.55218505859375, + "KL/mean": -600.6197509765625, + "KL/rejected_KL_mean": -772.687255859375, + "KL/std": 351.85308837890625, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0013857095036655664, + "fcm_dpo/delta": -0.08107978105545044, + "fcm_dpo/margin": 344.13507080078125, + "fcm_dpo/q_t": 0.38846635818481445, + "grad_norm": 29.150829315185547, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.7384995818138123, + "logits/rejected": -0.7426950931549072, + "logps/chosen": -482.79473876953125, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -857.7968139648438, + "loss": 1.0206, + "margin_dpo/margin_mean": 344.1350402832031, + "margin_dpo/margin_std": 349.2360534667969, + "step": 457 + }, + { + "KL/chosen_KL_mean": -439.2857971191406, + "KL/mean": -593.857421875, + "KL/rejected_KL_mean": -748.4290771484375, + "KL/std": 327.16412353515625, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.001371846185065806, + "fcm_dpo/delta": -0.02574240043759346, + "fcm_dpo/margin": 309.143310546875, + "fcm_dpo/q_t": 0.40318751335144043, + "grad_norm": 26.84882926940918, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.7498364448547363, + "logits/rejected": -0.7468490600585938, + "logps/chosen": -494.6946716308594, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -846.1123657226562, + "loss": 1.0682, + "margin_dpo/margin_mean": 309.143310546875, + "margin_dpo/margin_std": 370.48883056640625, + "step": 458 + }, + { + "KL/chosen_KL_mean": -463.2423400878906, + "KL/mean": -629.1153564453125, + "KL/rejected_KL_mean": -794.98828125, + "KL/std": 372.5406799316406, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0013608218869194388, + "fcm_dpo/delta": -0.054299160838127136, + "fcm_dpo/margin": 331.7459716796875, + "fcm_dpo/q_t": 0.3964211940765381, + "grad_norm": 29.428131103515625, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.7642187476158142, + "logits/rejected": -0.7846301794052124, + "logps/chosen": -509.7998352050781, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -881.1568603515625, + "loss": 1.0594, + "margin_dpo/margin_mean": 331.7460021972656, + "margin_dpo/margin_std": 412.528076171875, + "step": 459 + }, + { + "KL/chosen_KL_mean": -481.026123046875, + "KL/mean": -658.7969970703125, + "KL/rejected_KL_mean": -836.5679931640625, + "KL/std": 350.35693359375, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0013462214265018702, + "fcm_dpo/delta": -0.08253506571054459, + "fcm_dpo/margin": 355.5418395996094, + "fcm_dpo/q_t": 0.3898261487483978, + "grad_norm": 39.38566589355469, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.7451387643814087, + "logits/rejected": -0.7743414044380188, + "logps/chosen": -532.6610107421875, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -940.6873168945312, + "loss": 1.0254, + "margin_dpo/margin_mean": 355.5418395996094, + "margin_dpo/margin_std": 386.9627685546875, + "step": 460 + }, + { + "KL/chosen_KL_mean": -503.436767578125, + "KL/mean": -645.2528076171875, + "KL/rejected_KL_mean": -787.06884765625, + "KL/std": 352.14361572265625, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0013468009419739246, + "fcm_dpo/delta": 0.0185104850679636, + "fcm_dpo/margin": 283.632080078125, + "fcm_dpo/q_t": 0.4130192995071411, + "grad_norm": 28.407657623291016, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.746825098991394, + "logits/rejected": -0.7578277587890625, + "logps/chosen": -558.6187133789062, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -873.5457153320312, + "loss": 1.1045, + "margin_dpo/margin_mean": 283.632080078125, + "margin_dpo/margin_std": 391.5819091796875, + "step": 461 + }, + { + "KL/chosen_KL_mean": -505.7786865234375, + "KL/mean": -620.09814453125, + "KL/rejected_KL_mean": -734.41748046875, + "KL/std": 348.0482177734375, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.0013615848729386926, + "fcm_dpo/delta": 0.0916062593460083, + "fcm_dpo/margin": 228.63882446289062, + "fcm_dpo/q_t": 0.4293443262577057, + "grad_norm": 43.127281188964844, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.8342339396476746, + "logits/rejected": -0.8238426446914673, + "logps/chosen": -575.7067260742188, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -813.2586669921875, + "loss": 1.1633, + "margin_dpo/margin_mean": 228.63880920410156, + "margin_dpo/margin_std": 382.78851318359375, + "step": 462 + }, + { + "KL/chosen_KL_mean": -498.87615966796875, + "KL/mean": -656.1007690429688, + "KL/rejected_KL_mean": -813.3253173828125, + "KL/std": 371.81231689453125, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.0013673609355464578, + "fcm_dpo/delta": -0.031434111297130585, + "fcm_dpo/margin": 314.44915771484375, + "fcm_dpo/q_t": 0.4023106098175049, + "grad_norm": 33.80094528198242, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.7880203723907471, + "logits/rejected": -0.7993298768997192, + "logps/chosen": -554.1505126953125, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -902.350341796875, + "loss": 1.0776, + "margin_dpo/margin_mean": 314.44915771484375, + "margin_dpo/margin_std": 415.220458984375, + "step": 463 + }, + { + "KL/chosen_KL_mean": -484.74346923828125, + "KL/mean": -715.362548828125, + "KL/rejected_KL_mean": -945.9815673828125, + "KL/std": 432.91680908203125, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0013096630573272705, + "fcm_dpo/delta": -0.2190845012664795, + "fcm_dpo/margin": 461.2380676269531, + "fcm_dpo/q_t": 0.3623669147491455, + "grad_norm": 43.72697067260742, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.7602046728134155, + "logits/rejected": -0.8203283548355103, + "logps/chosen": -535.65576171875, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1048.470947265625, + "loss": 0.9457, + "margin_dpo/margin_mean": 461.23809814453125, + "margin_dpo/margin_std": 446.27789306640625, + "step": 464 + }, + { + "KL/chosen_KL_mean": -483.16943359375, + "KL/mean": -671.005126953125, + "KL/rejected_KL_mean": -858.8406982421875, + "KL/std": 426.2965087890625, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0012897943379357457, + "fcm_dpo/delta": -0.08878631144762039, + "fcm_dpo/margin": 375.67132568359375, + "fcm_dpo/q_t": 0.38684332370758057, + "grad_norm": 30.77594566345215, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.762598991394043, + "logits/rejected": -0.7830005884170532, + "logps/chosen": -543.2862548828125, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -972.7867431640625, + "loss": 1.0364, + "margin_dpo/margin_mean": 375.67132568359375, + "margin_dpo/margin_std": 442.65911865234375, + "step": 465 + }, + { + "KL/chosen_KL_mean": -537.658203125, + "KL/mean": -703.524658203125, + "KL/rejected_KL_mean": -869.3910522460938, + "KL/std": 399.0343933105469, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0012693159515038133, + "fcm_dpo/delta": -0.022521935403347015, + "fcm_dpo/margin": 331.73291015625, + "fcm_dpo/q_t": 0.4024147689342499, + "grad_norm": 28.25657844543457, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.8004381060600281, + "logits/rejected": -0.8068301677703857, + "logps/chosen": -590.5791015625, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -959.70654296875, + "loss": 1.0803, + "margin_dpo/margin_mean": 331.73291015625, + "margin_dpo/margin_std": 429.1689147949219, + "step": 466 + }, + { + "KL/chosen_KL_mean": -688.7811279296875, + "KL/mean": -858.5699462890625, + "KL/rejected_KL_mean": -1028.3587646484375, + "KL/std": 523.635986328125, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.001258077216334641, + "fcm_dpo/delta": -0.029918476939201355, + "fcm_dpo/margin": 339.57769775390625, + "fcm_dpo/q_t": 0.40674054622650146, + "grad_norm": 38.077091217041016, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.8847167491912842, + "logits/rejected": -0.8770743608474731, + "logps/chosen": -767.4969482421875, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1131.218994140625, + "loss": 1.1419, + "margin_dpo/margin_mean": 339.5777282714844, + "margin_dpo/margin_std": 610.0406494140625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -542.9632568359375, + "KL/mean": -752.2633666992188, + "KL/rejected_KL_mean": -961.5634765625, + "KL/std": 465.0583190917969, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0012406179448589683, + "fcm_dpo/delta": -0.1260991394519806, + "fcm_dpo/margin": 418.6001892089844, + "fcm_dpo/q_t": 0.3841235637664795, + "grad_norm": 61.846805572509766, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.8312078714370728, + "logits/rejected": -0.8401369452476501, + "logps/chosen": -612.898681640625, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1062.59228515625, + "loss": 1.0224, + "margin_dpo/margin_mean": 418.6001892089844, + "margin_dpo/margin_std": 518.4407958984375, + "step": 468 + }, + { + "KL/chosen_KL_mean": -593.6708984375, + "KL/mean": -812.1151123046875, + "KL/rejected_KL_mean": -1030.5592041015625, + "KL/std": 431.4645080566406, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0012034199899062514, + "fcm_dpo/delta": -0.13411511480808258, + "fcm_dpo/margin": 436.8883972167969, + "fcm_dpo/q_t": 0.379710853099823, + "grad_norm": 49.275856018066406, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.8810113668441772, + "logits/rejected": -0.8964806795120239, + "logps/chosen": -661.7955932617188, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1135.345703125, + "loss": 1.0045, + "margin_dpo/margin_mean": 436.888427734375, + "margin_dpo/margin_std": 475.6728820800781, + "step": 469 + }, + { + "KL/chosen_KL_mean": -513.2971801757812, + "KL/mean": -709.1964111328125, + "KL/rejected_KL_mean": -905.0955810546875, + "KL/std": 462.72998046875, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0011927220039069653, + "fcm_dpo/delta": -0.07057318836450577, + "fcm_dpo/margin": 391.79833984375, + "fcm_dpo/q_t": 0.3928372859954834, + "grad_norm": 30.90557289123535, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.8643077611923218, + "logits/rejected": -0.8792308568954468, + "logps/chosen": -557.089111328125, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -987.7984619140625, + "loss": 1.0563, + "margin_dpo/margin_mean": 391.7983703613281, + "margin_dpo/margin_std": 504.0444030761719, + "step": 470 + }, + { + "KL/chosen_KL_mean": -651.425537109375, + "KL/mean": -818.7632446289062, + "KL/rejected_KL_mean": -986.1010131835938, + "KL/std": 473.40313720703125, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0011793032754212618, + "fcm_dpo/delta": 0.004935339093208313, + "fcm_dpo/margin": 334.67547607421875, + "fcm_dpo/q_t": 0.41236090660095215, + "grad_norm": 43.108699798583984, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.9469370245933533, + "logits/rejected": -0.960279643535614, + "logps/chosen": -714.7650146484375, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1069.71142578125, + "loss": 1.1131, + "margin_dpo/margin_mean": 334.67547607421875, + "margin_dpo/margin_std": 508.63470458984375, + "step": 471 + }, + { + "KL/chosen_KL_mean": -689.9503784179688, + "KL/mean": -887.7750854492188, + "KL/rejected_KL_mean": -1085.599853515625, + "KL/std": 586.6046142578125, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0011702768970280886, + "fcm_dpo/delta": -0.06657374650239944, + "fcm_dpo/margin": 395.6494140625, + "fcm_dpo/q_t": 0.40174001455307007, + "grad_norm": 40.19095230102539, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.9527557492256165, + "logits/rejected": -0.9311193227767944, + "logps/chosen": -773.616455078125, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1202.80908203125, + "loss": 1.1006, + "margin_dpo/margin_mean": 395.6494140625, + "margin_dpo/margin_std": 633.9034423828125, + "step": 472 + }, + { + "KL/chosen_KL_mean": -824.57177734375, + "KL/mean": -919.5787353515625, + "KL/rejected_KL_mean": -1014.585693359375, + "KL/std": 616.15771484375, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0011985452147200704, + "fcm_dpo/delta": 0.176605686545372, + "fcm_dpo/margin": 190.01385498046875, + "fcm_dpo/q_t": 0.45228853821754456, + "grad_norm": 116.82813262939453, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.9687016010284424, + "logits/rejected": -0.9434047937393188, + "logps/chosen": -888.0687255859375, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1095.7322998046875, + "loss": 1.3663, + "margin_dpo/margin_mean": 190.01385498046875, + "margin_dpo/margin_std": 806.6006469726562, + "step": 473 + }, + { + "KL/chosen_KL_mean": -674.6010131835938, + "KL/mean": -844.0657958984375, + "KL/rejected_KL_mean": -1013.530517578125, + "KL/std": 521.8681640625, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.001220868667587638, + "fcm_dpo/delta": -0.015960246324539185, + "fcm_dpo/margin": 338.929443359375, + "fcm_dpo/q_t": 0.4060695767402649, + "grad_norm": 48.320919036865234, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.9456368684768677, + "logits/rejected": -0.9505201578140259, + "logps/chosen": -727.2129516601562, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1103.61083984375, + "loss": 1.1324, + "margin_dpo/margin_mean": 338.929443359375, + "margin_dpo/margin_std": 563.3499145507812, + "step": 474 + }, + { + "KL/chosen_KL_mean": -497.0267639160156, + "KL/mean": -710.57470703125, + "KL/rejected_KL_mean": -924.12255859375, + "KL/std": 424.65301513671875, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0011858819052577019, + "fcm_dpo/delta": -0.11253334581851959, + "fcm_dpo/margin": 427.09576416015625, + "fcm_dpo/q_t": 0.383480429649353, + "grad_norm": 42.11455154418945, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.9882975816726685, + "logits/rejected": -1.0178190469741821, + "logps/chosen": -539.52197265625, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1014.1854858398438, + "loss": 1.0089, + "margin_dpo/margin_mean": 427.09576416015625, + "margin_dpo/margin_std": 453.58563232421875, + "step": 475 + }, + { + "KL/chosen_KL_mean": -604.0947265625, + "KL/mean": -782.0799560546875, + "KL/rejected_KL_mean": -960.065185546875, + "KL/std": 476.6460876464844, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.0011776951141655445, + "fcm_dpo/delta": -0.02007940784096718, + "fcm_dpo/margin": 355.97052001953125, + "fcm_dpo/q_t": 0.4050132632255554, + "grad_norm": 57.491458892822266, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.9728083610534668, + "logits/rejected": -0.9907442331314087, + "logps/chosen": -647.0440673828125, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1033.775390625, + "loss": 1.0816, + "margin_dpo/margin_mean": 355.97052001953125, + "margin_dpo/margin_std": 475.0918884277344, + "step": 476 + }, + { + "KL/chosen_KL_mean": -647.10986328125, + "KL/mean": -821.5865478515625, + "KL/rejected_KL_mean": -996.063232421875, + "KL/std": 505.9287109375, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0011787350522354245, + "fcm_dpo/delta": -0.011983605101704597, + "fcm_dpo/margin": 348.95343017578125, + "fcm_dpo/q_t": 0.4073077440261841, + "grad_norm": 35.119571685791016, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -1.0101265907287598, + "logits/rejected": -0.9804463386535645, + "logps/chosen": -717.8824462890625, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1072.20068359375, + "loss": 1.1265, + "margin_dpo/margin_mean": 348.95343017578125, + "margin_dpo/margin_std": 587.4610595703125, + "step": 477 + }, + { + "KL/chosen_KL_mean": -534.1758422851562, + "KL/mean": -737.901611328125, + "KL/rejected_KL_mean": -941.6273193359375, + "KL/std": 453.39263916015625, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.0011632876703515649, + "fcm_dpo/delta": -0.07759275287389755, + "fcm_dpo/margin": 407.4515380859375, + "fcm_dpo/q_t": 0.3934464752674103, + "grad_norm": 40.0849609375, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.8814162611961365, + "logits/rejected": -0.9091761112213135, + "logps/chosen": -575.6163330078125, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1026.9892578125, + "loss": 1.0564, + "margin_dpo/margin_mean": 407.4515380859375, + "margin_dpo/margin_std": 538.770263671875, + "step": 478 + }, + { + "KL/chosen_KL_mean": -686.47119140625, + "KL/mean": -880.3046875, + "KL/rejected_KL_mean": -1074.13818359375, + "KL/std": 546.978515625, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0011555879609659314, + "fcm_dpo/delta": -0.050895098596811295, + "fcm_dpo/margin": 387.66693115234375, + "fcm_dpo/q_t": 0.403221070766449, + "grad_norm": 34.835323333740234, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.954513669013977, + "logits/rejected": -0.9876626133918762, + "logps/chosen": -740.379150390625, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1169.2545166015625, + "loss": 1.1081, + "margin_dpo/margin_mean": 387.66693115234375, + "margin_dpo/margin_std": 637.2411499023438, + "step": 479 + }, + { + "KL/chosen_KL_mean": -810.0494995117188, + "KL/mean": -961.0078735351562, + "KL/rejected_KL_mean": -1111.96630859375, + "KL/std": 510.32623291015625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0011639440199360251, + "fcm_dpo/delta": 0.04795617610216141, + "fcm_dpo/margin": 301.916748046875, + "fcm_dpo/q_t": 0.42261946201324463, + "grad_norm": 64.39877319335938, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.9891233444213867, + "logits/rejected": -0.990409255027771, + "logps/chosen": -868.732177734375, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1194.898681640625, + "loss": 1.1746, + "margin_dpo/margin_mean": 301.916748046875, + "margin_dpo/margin_std": 555.5850830078125, + "step": 480 + }, + { + "KL/chosen_KL_mean": -670.7325439453125, + "KL/mean": -898.3741455078125, + "KL/rejected_KL_mean": -1126.0157470703125, + "KL/std": 529.572998046875, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0011354672024026513, + "fcm_dpo/delta": -0.12324470281600952, + "fcm_dpo/margin": 455.2831726074219, + "fcm_dpo/q_t": 0.3845733404159546, + "grad_norm": 44.59912872314453, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -1.014250636100769, + "logits/rejected": -1.0467482805252075, + "logps/chosen": -725.6968383789062, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1218.436279296875, + "loss": 1.0271, + "margin_dpo/margin_mean": 455.2831726074219, + "margin_dpo/margin_std": 567.700439453125, + "step": 481 + }, + { + "KL/chosen_KL_mean": -728.3932495117188, + "KL/mean": -830.1834716796875, + "KL/rejected_KL_mean": -931.9737548828125, + "KL/std": 535.198974609375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0011569273192435503, + "fcm_dpo/delta": 0.1683010309934616, + "fcm_dpo/margin": 203.58041381835938, + "fcm_dpo/q_t": 0.4449790120124817, + "grad_norm": 65.31682586669922, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.9308723211288452, + "logits/rejected": -0.9238674640655518, + "logps/chosen": -795.9467163085938, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1019.563232421875, + "loss": 1.2815, + "margin_dpo/margin_mean": 203.58041381835938, + "margin_dpo/margin_std": 618.666015625, + "step": 482 + }, + { + "KL/chosen_KL_mean": -625.8432006835938, + "KL/mean": -861.8372802734375, + "KL/rejected_KL_mean": -1097.831298828125, + "KL/std": 528.1412353515625, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.001138403546065092, + "fcm_dpo/delta": -0.14554277062416077, + "fcm_dpo/margin": 471.9882507324219, + "fcm_dpo/q_t": 0.38148680329322815, + "grad_norm": 33.437992095947266, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.9590774774551392, + "logits/rejected": -0.9771822690963745, + "logps/chosen": -689.1729736328125, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1193.618408203125, + "loss": 1.0151, + "margin_dpo/margin_mean": 471.98822021484375, + "margin_dpo/margin_std": 586.5374755859375, + "step": 483 + }, + { + "KL/chosen_KL_mean": -522.0122680664062, + "KL/mean": -734.5303344726562, + "KL/rejected_KL_mean": -947.04833984375, + "KL/std": 494.6111145019531, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0011234726989641786, + "fcm_dpo/delta": -0.08141546696424484, + "fcm_dpo/margin": 425.0361328125, + "fcm_dpo/q_t": 0.39181143045425415, + "grad_norm": 63.34284973144531, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.9345070123672485, + "logits/rejected": -0.9631925225257874, + "logps/chosen": -581.150390625, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1031.4197998046875, + "loss": 1.039, + "margin_dpo/margin_mean": 425.0361633300781, + "margin_dpo/margin_std": 503.9326171875, + "step": 484 + }, + { + "KL/chosen_KL_mean": -554.0560302734375, + "KL/mean": -758.105224609375, + "KL/rejected_KL_mean": -962.1544189453125, + "KL/std": 499.4193115234375, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0011075625661760569, + "fcm_dpo/delta": -0.05443325266242027, + "fcm_dpo/margin": 408.098388671875, + "fcm_dpo/q_t": 0.39861178398132324, + "grad_norm": 36.1124153137207, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.8819291591644287, + "logits/rejected": -0.9065327644348145, + "logps/chosen": -612.9055786132812, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1065.5185546875, + "loss": 1.0712, + "margin_dpo/margin_mean": 408.098388671875, + "margin_dpo/margin_std": 557.530517578125, + "step": 485 + }, + { + "KL/chosen_KL_mean": -628.2642211914062, + "KL/mean": -859.0435180664062, + "KL/rejected_KL_mean": -1089.82275390625, + "KL/std": 558.32421875, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.001084483927115798, + "fcm_dpo/delta": -0.10589458048343658, + "fcm_dpo/margin": 461.55859375, + "fcm_dpo/q_t": 0.3902357220649719, + "grad_norm": 39.234439849853516, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.9813936948776245, + "logits/rejected": -1.007172703742981, + "logps/chosen": -683.52392578125, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1181.962158203125, + "loss": 1.0667, + "margin_dpo/margin_mean": 461.55859375, + "margin_dpo/margin_std": 680.8170166015625, + "step": 486 + }, + { + "KL/chosen_KL_mean": -680.6168212890625, + "KL/mean": -849.6842041015625, + "KL/rejected_KL_mean": -1018.7515258789062, + "KL/std": 516.2461547851562, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0010883330833166838, + "fcm_dpo/delta": 0.03281679004430771, + "fcm_dpo/margin": 338.1346435546875, + "fcm_dpo/q_t": 0.41458362340927124, + "grad_norm": 45.096126556396484, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.9377965927124023, + "logits/rejected": -0.9469963312149048, + "logps/chosen": -733.68017578125, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1111.17041015625, + "loss": 1.1299, + "margin_dpo/margin_mean": 338.1346435546875, + "margin_dpo/margin_std": 531.5714111328125, + "step": 487 + }, + { + "KL/chosen_KL_mean": -541.6302490234375, + "KL/mean": -715.9400634765625, + "KL/rejected_KL_mean": -890.2498779296875, + "KL/std": 455.6866149902344, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0010932320728898048, + "fcm_dpo/delta": 0.019282767549157143, + "fcm_dpo/margin": 348.61956787109375, + "fcm_dpo/q_t": 0.4124680161476135, + "grad_norm": 39.201820373535156, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.8679413795471191, + "logits/rejected": -0.8775743842124939, + "logps/chosen": -593.8583984375, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -974.2564086914062, + "loss": 1.1001, + "margin_dpo/margin_mean": 348.61956787109375, + "margin_dpo/margin_std": 459.080810546875, + "step": 488 + }, + { + "KL/chosen_KL_mean": -514.0245971679688, + "KL/mean": -688.5931396484375, + "KL/rejected_KL_mean": -863.1616821289062, + "KL/std": 459.21954345703125, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0010969855356961489, + "fcm_dpo/delta": 0.017366131767630577, + "fcm_dpo/margin": 349.1370849609375, + "fcm_dpo/q_t": 0.4141993820667267, + "grad_norm": 37.24169921875, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.905672550201416, + "logits/rejected": -0.910228431224823, + "logps/chosen": -570.0142211914062, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -942.559814453125, + "loss": 1.1111, + "margin_dpo/margin_mean": 349.1370849609375, + "margin_dpo/margin_std": 508.865234375, + "step": 489 + }, + { + "KL/chosen_KL_mean": -622.357421875, + "KL/mean": -807.6400146484375, + "KL/rejected_KL_mean": -992.922607421875, + "KL/std": 559.6922607421875, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0010955492034554482, + "fcm_dpo/delta": -0.006315100938081741, + "fcm_dpo/margin": 370.56524658203125, + "fcm_dpo/q_t": 0.4109686017036438, + "grad_norm": 45.62641906738281, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.9424386024475098, + "logits/rejected": -0.9847195148468018, + "logps/chosen": -674.7237548828125, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1103.3316650390625, + "loss": 1.1407, + "margin_dpo/margin_mean": 370.56524658203125, + "margin_dpo/margin_std": 661.7091064453125, + "step": 490 + }, + { + "KL/chosen_KL_mean": -580.51904296875, + "KL/mean": -707.26123046875, + "KL/rejected_KL_mean": -834.0032958984375, + "KL/std": 474.60821533203125, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0011142883449792862, + "fcm_dpo/delta": 0.12073345482349396, + "fcm_dpo/margin": 253.4842987060547, + "fcm_dpo/q_t": 0.43588536977767944, + "grad_norm": 30.27996826171875, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.9451805353164673, + "logits/rejected": -0.9439833760261536, + "logps/chosen": -640.6353149414062, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -907.276123046875, + "loss": 1.1876, + "margin_dpo/margin_mean": 253.4842987060547, + "margin_dpo/margin_std": 468.6629333496094, + "step": 491 + }, + { + "KL/chosen_KL_mean": -613.3606567382812, + "KL/mean": -741.74755859375, + "KL/rejected_KL_mean": -870.1344604492188, + "KL/std": 496.1963806152344, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0011359485797584057, + "fcm_dpo/delta": 0.11171634495258331, + "fcm_dpo/margin": 256.7738037109375, + "fcm_dpo/q_t": 0.43545809388160706, + "grad_norm": 49.777320861816406, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.9806656837463379, + "logits/rejected": -0.9645482897758484, + "logps/chosen": -667.3548583984375, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -942.7940673828125, + "loss": 1.2169, + "margin_dpo/margin_mean": 256.7738037109375, + "margin_dpo/margin_std": 575.3779296875, + "step": 492 + }, + { + "KL/chosen_KL_mean": -648.8692626953125, + "KL/mean": -783.3594970703125, + "KL/rejected_KL_mean": -917.8497314453125, + "KL/std": 541.2496948242188, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0011525104055181146, + "fcm_dpo/delta": 0.09251011162996292, + "fcm_dpo/margin": 268.9804992675781, + "fcm_dpo/q_t": 0.42842549085617065, + "grad_norm": 38.381221771240234, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -1.0409237146377563, + "logits/rejected": -1.0164510011672974, + "logps/chosen": -724.366455078125, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1005.1727294921875, + "loss": 1.1866, + "margin_dpo/margin_mean": 268.9804992675781, + "margin_dpo/margin_std": 513.2163696289062, + "step": 493 + }, + { + "KL/chosen_KL_mean": -514.3545532226562, + "KL/mean": -737.381103515625, + "KL/rejected_KL_mean": -960.4078369140625, + "KL/std": 491.7872314453125, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.0011451728641986847, + "fcm_dpo/delta": -0.11687131971120834, + "fcm_dpo/margin": 446.0532531738281, + "fcm_dpo/q_t": 0.38306695222854614, + "grad_norm": 50.34432601928711, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.8729780316352844, + "logits/rejected": -0.9104505777359009, + "logps/chosen": -555.7138061523438, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1046.4991455078125, + "loss": 1.0111, + "margin_dpo/margin_mean": 446.05328369140625, + "margin_dpo/margin_std": 489.10693359375, + "step": 494 + }, + { + "KL/chosen_KL_mean": -583.2138671875, + "KL/mean": -775.3177490234375, + "KL/rejected_KL_mean": -967.421630859375, + "KL/std": 498.6965637207031, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.001133341109380126, + "fcm_dpo/delta": -0.03704039007425308, + "fcm_dpo/margin": 384.20770263671875, + "fcm_dpo/q_t": 0.4044431447982788, + "grad_norm": 38.2929573059082, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -1.0106769800186157, + "logits/rejected": -1.0235304832458496, + "logps/chosen": -646.7489624023438, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1058.8460693359375, + "loss": 1.0954, + "margin_dpo/margin_mean": 384.20770263671875, + "margin_dpo/margin_std": 592.011962890625, + "step": 495 + }, + { + "KL/chosen_KL_mean": -713.7726440429688, + "KL/mean": -815.0147094726562, + "KL/rejected_KL_mean": -916.2567138671875, + "KL/std": 403.49005126953125, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0011600281577557325, + "fcm_dpo/delta": 0.1690365970134735, + "fcm_dpo/margin": 202.48403930664062, + "fcm_dpo/q_t": 0.44568806886672974, + "grad_norm": 62.622806549072266, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.949141263961792, + "logits/rejected": -0.9214235544204712, + "logps/chosen": -786.3646240234375, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1000.5860595703125, + "loss": 1.2333, + "margin_dpo/margin_mean": 202.48403930664062, + "margin_dpo/margin_std": 456.31549072265625, + "step": 496 + }, + { + "KL/chosen_KL_mean": -679.3038330078125, + "KL/mean": -779.9605102539062, + "KL/rejected_KL_mean": -880.6171875, + "KL/std": 490.28515625, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0011910130269825459, + "fcm_dpo/delta": 0.16437701880931854, + "fcm_dpo/margin": 201.3133087158203, + "fcm_dpo/q_t": 0.4456322491168976, + "grad_norm": 48.48257064819336, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.9629020690917969, + "logits/rejected": -0.9487000703811646, + "logps/chosen": -737.8978271484375, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -956.9055786132812, + "loss": 1.2377, + "margin_dpo/margin_mean": 201.3133087158203, + "margin_dpo/margin_std": 457.0677490234375, + "step": 497 + }, + { + "KL/chosen_KL_mean": -650.6729736328125, + "KL/mean": -820.2717895507812, + "KL/rejected_KL_mean": -989.87060546875, + "KL/std": 522.2001953125, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.001205613138154149, + "fcm_dpo/delta": -0.009361925534904003, + "fcm_dpo/margin": 339.19757080078125, + "fcm_dpo/q_t": 0.4081471562385559, + "grad_norm": 38.5706787109375, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.9384486675262451, + "logits/rejected": -0.9260154366493225, + "logps/chosen": -721.878662109375, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1073.82861328125, + "loss": 1.1342, + "margin_dpo/margin_mean": 339.19757080078125, + "margin_dpo/margin_std": 590.6138305664062, + "step": 498 + }, + { + "KL/chosen_KL_mean": -549.56298828125, + "KL/mean": -761.119140625, + "KL/rejected_KL_mean": -972.6754150390625, + "KL/std": 499.8108215332031, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0011859136866405606, + "fcm_dpo/delta": -0.10731419175863266, + "fcm_dpo/margin": 423.1124267578125, + "fcm_dpo/q_t": 0.3869936168193817, + "grad_norm": 44.039039611816406, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.9746659398078918, + "logits/rejected": -1.0103843212127686, + "logps/chosen": -600.818115234375, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1073.754150390625, + "loss": 1.0422, + "margin_dpo/margin_mean": 423.1124267578125, + "margin_dpo/margin_std": 553.3563842773438, + "step": 499 + }, + { + "KL/chosen_KL_mean": -646.4860229492188, + "KL/mean": -822.3896484375, + "KL/rejected_KL_mean": -998.2933349609375, + "KL/std": 441.0538330078125, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0011761856731027365, + "fcm_dpo/delta": -0.014477956108748913, + "fcm_dpo/margin": 351.80731201171875, + "fcm_dpo/q_t": 0.4063568711280823, + "grad_norm": 30.520992279052734, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.9440732002258301, + "logits/rejected": -0.9755163192749023, + "logps/chosen": -703.513427734375, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1092.2275390625, + "loss": 1.111, + "margin_dpo/margin_mean": 351.80731201171875, + "margin_dpo/margin_std": 543.197509765625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -566.9063110351562, + "KL/mean": -751.845947265625, + "KL/rejected_KL_mean": -936.7855834960938, + "KL/std": 465.94305419921875, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0011742108035832644, + "fcm_dpo/delta": -0.03591924160718918, + "fcm_dpo/margin": 369.8792419433594, + "fcm_dpo/q_t": 0.4037385582923889, + "grad_norm": 37.079898834228516, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.8725494146347046, + "logits/rejected": -0.8555170297622681, + "logps/chosen": -621.265869140625, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1016.9422607421875, + "loss": 1.1038, + "margin_dpo/margin_mean": 369.8792419433594, + "margin_dpo/margin_std": 577.346923828125, + "step": 501 + }, + { + "KL/chosen_KL_mean": -500.33978271484375, + "KL/mean": -687.7737426757812, + "KL/rejected_KL_mean": -875.2076416015625, + "KL/std": 423.86920166015625, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0011633912799879909, + "fcm_dpo/delta": -0.03774793818593025, + "fcm_dpo/margin": 374.867919921875, + "fcm_dpo/q_t": 0.4007149934768677, + "grad_norm": 37.201622009277344, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.8690605759620667, + "logits/rejected": -0.8578146696090698, + "logps/chosen": -567.9403076171875, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -958.1564331054688, + "loss": 1.0587, + "margin_dpo/margin_mean": 374.867919921875, + "margin_dpo/margin_std": 448.31671142578125, + "step": 502 + }, + { + "KL/chosen_KL_mean": -496.2381591796875, + "KL/mean": -664.9107666015625, + "KL/rejected_KL_mean": -833.5833129882812, + "KL/std": 393.21319580078125, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.0011592555092647672, + "fcm_dpo/delta": 0.009215807542204857, + "fcm_dpo/margin": 337.34515380859375, + "fcm_dpo/q_t": 0.41072437167167664, + "grad_norm": 26.517845153808594, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.8493194580078125, + "logits/rejected": -0.8390483260154724, + "logps/chosen": -551.3165283203125, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -916.0887451171875, + "loss": 1.0903, + "margin_dpo/margin_mean": 337.3451843261719, + "margin_dpo/margin_std": 431.22509765625, + "step": 503 + }, + { + "KL/chosen_KL_mean": -541.871826171875, + "KL/mean": -745.7672729492188, + "KL/rejected_KL_mean": -949.6627197265625, + "KL/std": 474.7845458984375, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0011516192462295294, + "fcm_dpo/delta": -0.07298626005649567, + "fcm_dpo/margin": 407.7908630371094, + "fcm_dpo/q_t": 0.39398640394210815, + "grad_norm": 32.1545524597168, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.9057151079177856, + "logits/rejected": -0.9302019476890564, + "logps/chosen": -601.8375854492188, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1053.4248046875, + "loss": 1.0539, + "margin_dpo/margin_mean": 407.7908630371094, + "margin_dpo/margin_std": 513.2054443359375, + "step": 504 + }, + { + "KL/chosen_KL_mean": -607.4645385742188, + "KL/mean": -718.3087158203125, + "KL/rejected_KL_mean": -829.1529541015625, + "KL/std": 453.771728515625, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0011712907580658793, + "fcm_dpo/delta": 0.14396202564239502, + "fcm_dpo/margin": 221.68844604492188, + "fcm_dpo/q_t": 0.4401799440383911, + "grad_norm": 35.92436981201172, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.9009963274002075, + "logits/rejected": -0.8728290796279907, + "logps/chosen": -683.6194458007812, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -917.7383422851562, + "loss": 1.2053, + "margin_dpo/margin_mean": 221.68844604492188, + "margin_dpo/margin_std": 429.1565856933594, + "step": 505 + }, + { + "KL/chosen_KL_mean": -479.02081298828125, + "KL/mean": -656.36962890625, + "KL/rejected_KL_mean": -833.7184448242188, + "KL/std": 438.58642578125, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0011714599095284939, + "fcm_dpo/delta": -0.016566481441259384, + "fcm_dpo/margin": 354.6976318359375, + "fcm_dpo/q_t": 0.4052731394767761, + "grad_norm": 33.494361877441406, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.8967812061309814, + "logits/rejected": -0.9098480939865112, + "logps/chosen": -527.9813232421875, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -912.133544921875, + "loss": 1.0881, + "margin_dpo/margin_mean": 354.6976318359375, + "margin_dpo/margin_std": 482.19573974609375, + "step": 506 + }, + { + "KL/chosen_KL_mean": -569.2562255859375, + "KL/mean": -719.5776977539062, + "KL/rejected_KL_mean": -869.8992309570312, + "KL/std": 500.6405334472656, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0011815531179308891, + "fcm_dpo/delta": 0.0464482381939888, + "fcm_dpo/margin": 300.6429443359375, + "fcm_dpo/q_t": 0.4232487082481384, + "grad_norm": 30.135711669921875, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.9168051481246948, + "logits/rejected": -0.9183909893035889, + "logps/chosen": -636.0577392578125, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -965.2720947265625, + "loss": 1.1575, + "margin_dpo/margin_mean": 300.6429443359375, + "margin_dpo/margin_std": 555.3529052734375, + "step": 507 + }, + { + "KL/chosen_KL_mean": -595.1732177734375, + "KL/mean": -726.621826171875, + "KL/rejected_KL_mean": -858.0704956054688, + "KL/std": 493.1138000488281, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0012043035821989179, + "fcm_dpo/delta": 0.0857793465256691, + "fcm_dpo/margin": 262.8973083496094, + "fcm_dpo/q_t": 0.4305441081523895, + "grad_norm": 35.5110969543457, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.8891359567642212, + "logits/rejected": -0.888923704624176, + "logps/chosen": -666.4766845703125, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -953.6980590820312, + "loss": 1.2047, + "margin_dpo/margin_mean": 262.89727783203125, + "margin_dpo/margin_std": 571.6166381835938, + "step": 508 + }, + { + "KL/chosen_KL_mean": -461.79888916015625, + "KL/mean": -610.6336669921875, + "KL/rejected_KL_mean": -759.4684448242188, + "KL/std": 369.9573974609375, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0012186921667307615, + "fcm_dpo/delta": 0.03808768093585968, + "fcm_dpo/margin": 297.6695251464844, + "fcm_dpo/q_t": 0.41801512241363525, + "grad_norm": 30.8708438873291, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.8370890617370605, + "logits/rejected": -0.8362618684768677, + "logps/chosen": -525.6178588867188, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -842.7249145507812, + "loss": 1.1287, + "margin_dpo/margin_mean": 297.6695251464844, + "margin_dpo/margin_std": 453.9029235839844, + "step": 509 + }, + { + "KL/chosen_KL_mean": -563.912841796875, + "KL/mean": -755.1748046875, + "KL/rejected_KL_mean": -946.4368896484375, + "KL/std": 429.85711669921875, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0012061686720699072, + "fcm_dpo/delta": -0.06430751830339432, + "fcm_dpo/margin": 382.52410888671875, + "fcm_dpo/q_t": 0.39263206720352173, + "grad_norm": 28.540021896362305, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.8131314516067505, + "logits/rejected": -0.8310078382492065, + "logps/chosen": -615.791259765625, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1049.2020263671875, + "loss": 1.0378, + "margin_dpo/margin_mean": 382.52410888671875, + "margin_dpo/margin_std": 421.2060241699219, + "step": 510 + }, + { + "KL/chosen_KL_mean": -511.52386474609375, + "KL/mean": -663.81982421875, + "KL/rejected_KL_mean": -816.11572265625, + "KL/std": 450.66741943359375, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0012070488883182406, + "fcm_dpo/delta": 0.033546678721904755, + "fcm_dpo/margin": 304.5919189453125, + "fcm_dpo/q_t": 0.41673213243484497, + "grad_norm": 39.95866394042969, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.887006402015686, + "logits/rejected": -0.8925356864929199, + "logps/chosen": -571.761962890625, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -908.9725341796875, + "loss": 1.1499, + "margin_dpo/margin_mean": 304.5918884277344, + "margin_dpo/margin_std": 543.8143310546875, + "step": 511 + }, + { + "KL/chosen_KL_mean": -415.99591064453125, + "KL/mean": -618.7208251953125, + "KL/rejected_KL_mean": -821.4456787109375, + "KL/std": 406.76507568359375, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.0011907469015568495, + "fcm_dpo/delta": -0.0871933251619339, + "fcm_dpo/margin": 405.4497985839844, + "fcm_dpo/q_t": 0.3876630663871765, + "grad_norm": 48.958831787109375, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.8688769340515137, + "logits/rejected": -0.8867466449737549, + "logps/chosen": -470.9013977050781, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -903.3215942382812, + "loss": 1.0166, + "margin_dpo/margin_mean": 405.4497985839844, + "margin_dpo/margin_std": 413.68243408203125, + "step": 512 + }, + { + "KL/chosen_KL_mean": -532.3680419921875, + "KL/mean": -676.462158203125, + "KL/rejected_KL_mean": -820.5563354492188, + "KL/std": 391.255859375, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0011904649436473846, + "fcm_dpo/delta": 0.05852006375789642, + "fcm_dpo/margin": 288.18829345703125, + "fcm_dpo/q_t": 0.42187637090682983, + "grad_norm": 43.607234954833984, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.9294281601905823, + "logits/rejected": -0.9153304696083069, + "logps/chosen": -597.2844848632812, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -896.6187744140625, + "loss": 1.144, + "margin_dpo/margin_mean": 288.18829345703125, + "margin_dpo/margin_std": 450.1658630371094, + "step": 513 + }, + { + "KL/chosen_KL_mean": -560.2642822265625, + "KL/mean": -704.6409912109375, + "KL/rejected_KL_mean": -849.0177001953125, + "KL/std": 436.6456298828125, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.0012128644157201052, + "fcm_dpo/delta": 0.05142746493220329, + "fcm_dpo/margin": 288.75347900390625, + "fcm_dpo/q_t": 0.42241793870925903, + "grad_norm": 30.854013442993164, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.8636369109153748, + "logits/rejected": -0.85251784324646, + "logps/chosen": -634.4937744140625, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -927.9632568359375, + "loss": 1.1465, + "margin_dpo/margin_mean": 288.75347900390625, + "margin_dpo/margin_std": 489.2330322265625, + "step": 514 + }, + { + "KL/chosen_KL_mean": -462.2237243652344, + "KL/mean": -578.16015625, + "KL/rejected_KL_mean": -694.0965576171875, + "KL/std": 367.974365234375, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0012354489881545305, + "fcm_dpo/delta": 0.116858571767807, + "fcm_dpo/margin": 231.8728485107422, + "fcm_dpo/q_t": 0.4326532483100891, + "grad_norm": 42.6376953125, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.8454920649528503, + "logits/rejected": -0.857939600944519, + "logps/chosen": -512.6253051757812, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -781.1943359375, + "loss": 1.1862, + "margin_dpo/margin_mean": 231.87286376953125, + "margin_dpo/margin_std": 428.89862060546875, + "step": 515 + }, + { + "KL/chosen_KL_mean": -507.8172607421875, + "KL/mean": -683.4832763671875, + "KL/rejected_KL_mean": -859.1492919921875, + "KL/std": 429.0656433105469, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.0012392147909849882, + "fcm_dpo/delta": -0.03704463690519333, + "fcm_dpo/margin": 351.33203125, + "fcm_dpo/q_t": 0.3985579013824463, + "grad_norm": 41.602333068847656, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.9168685674667358, + "logits/rejected": -0.9448647499084473, + "logps/chosen": -572.466796875, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -970.8717041015625, + "loss": 1.0616, + "margin_dpo/margin_mean": 351.33203125, + "margin_dpo/margin_std": 418.1084289550781, + "step": 516 + }, + { + "KL/chosen_KL_mean": -522.5617065429688, + "KL/mean": -723.7996826171875, + "KL/rejected_KL_mean": -925.03759765625, + "KL/std": 468.8752136230469, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0012140416074544191, + "fcm_dpo/delta": -0.09342710673809052, + "fcm_dpo/margin": 402.4759521484375, + "fcm_dpo/q_t": 0.3916882276535034, + "grad_norm": 28.097633361816406, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.8645687103271484, + "logits/rejected": -0.8641697764396667, + "logps/chosen": -583.4752197265625, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1014.1206665039062, + "loss": 1.0422, + "margin_dpo/margin_mean": 402.4759521484375, + "margin_dpo/margin_std": 517.1727294921875, + "step": 517 + }, + { + "KL/chosen_KL_mean": -483.2095642089844, + "KL/mean": -649.2745971679688, + "KL/rejected_KL_mean": -815.3397216796875, + "KL/std": 384.469482421875, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.0012063465546816587, + "fcm_dpo/delta": -0.0009220875799655914, + "fcm_dpo/margin": 332.13018798828125, + "fcm_dpo/q_t": 0.4069374203681946, + "grad_norm": 48.325042724609375, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.8562802076339722, + "logits/rejected": -0.8714978098869324, + "logps/chosen": -540.6654052734375, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -900.6524047851562, + "loss": 1.0868, + "margin_dpo/margin_mean": 332.1301574707031, + "margin_dpo/margin_std": 420.98101806640625, + "step": 518 + }, + { + "KL/chosen_KL_mean": -485.0940856933594, + "KL/mean": -603.6209106445312, + "KL/rejected_KL_mean": -722.147705078125, + "KL/std": 347.04144287109375, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.001234774012118578, + "fcm_dpo/delta": 0.1098669171333313, + "fcm_dpo/margin": 237.0536346435547, + "fcm_dpo/q_t": 0.432472825050354, + "grad_norm": 37.98015213012695, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.850660502910614, + "logits/rejected": -0.8336910009384155, + "logps/chosen": -559.1574096679688, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -826.5919189453125, + "loss": 1.1852, + "margin_dpo/margin_mean": 237.0536346435547, + "margin_dpo/margin_std": 436.78704833984375, + "step": 519 + }, + { + "KL/chosen_KL_mean": -529.6180419921875, + "KL/mean": -687.3538818359375, + "KL/rejected_KL_mean": -845.0897216796875, + "KL/std": 419.5048522949219, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.0012346227886155248, + "fcm_dpo/delta": 0.010699015110731125, + "fcm_dpo/margin": 315.4716796875, + "fcm_dpo/q_t": 0.413169264793396, + "grad_norm": 43.05133819580078, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.8334793448448181, + "logits/rejected": -0.8424146175384521, + "logps/chosen": -599.9179077148438, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -945.071044921875, + "loss": 1.1148, + "margin_dpo/margin_mean": 315.4716796875, + "margin_dpo/margin_std": 467.21124267578125, + "step": 520 + }, + { + "KL/chosen_KL_mean": -461.21221923828125, + "KL/mean": -627.90380859375, + "KL/rejected_KL_mean": -794.5953369140625, + "KL/std": 424.56756591796875, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0012456991244107485, + "fcm_dpo/delta": -0.01689179241657257, + "fcm_dpo/margin": 333.383056640625, + "fcm_dpo/q_t": 0.40557414293289185, + "grad_norm": 31.14696502685547, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.8552919626235962, + "logits/rejected": -0.8607101440429688, + "logps/chosen": -519.3551635742188, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -877.8759155273438, + "loss": 1.0865, + "margin_dpo/margin_mean": 333.383056640625, + "margin_dpo/margin_std": 437.95452880859375, + "step": 521 + }, + { + "KL/chosen_KL_mean": -543.6200561523438, + "KL/mean": -694.7198486328125, + "KL/rejected_KL_mean": -845.8197021484375, + "KL/std": 454.48236083984375, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0012406579917296767, + "fcm_dpo/delta": 0.02604612335562706, + "fcm_dpo/margin": 302.1996154785156, + "fcm_dpo/q_t": 0.4148826003074646, + "grad_norm": 36.55927658081055, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.9067702293395996, + "logits/rejected": -0.9117075204849243, + "logps/chosen": -590.167724609375, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -911.8335571289062, + "loss": 1.1287, + "margin_dpo/margin_mean": 302.1996154785156, + "margin_dpo/margin_std": 479.10186767578125, + "step": 522 + }, + { + "KL/chosen_KL_mean": -579.4052734375, + "KL/mean": -743.9030151367188, + "KL/rejected_KL_mean": -908.4006958007812, + "KL/std": 517.8148193359375, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0012396316742524505, + "fcm_dpo/delta": -0.008185341022908688, + "fcm_dpo/margin": 328.995361328125, + "fcm_dpo/q_t": 0.4095180630683899, + "grad_norm": 41.75992965698242, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.9010941982269287, + "logits/rejected": -0.9034140110015869, + "logps/chosen": -641.1749267578125, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -992.162109375, + "loss": 1.1224, + "margin_dpo/margin_mean": 328.995361328125, + "margin_dpo/margin_std": 542.5953979492188, + "step": 523 + }, + { + "KL/chosen_KL_mean": -565.06201171875, + "KL/mean": -733.1220703125, + "KL/rejected_KL_mean": -901.18212890625, + "KL/std": 478.7225341796875, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0012296756030991673, + "fcm_dpo/delta": -0.014783322811126709, + "fcm_dpo/margin": 336.12017822265625, + "fcm_dpo/q_t": 0.4039151668548584, + "grad_norm": 43.43800354003906, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.982138991355896, + "logits/rejected": -0.9809105396270752, + "logps/chosen": -643.134033203125, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -982.484130859375, + "loss": 1.1057, + "margin_dpo/margin_mean": 336.1201477050781, + "margin_dpo/margin_std": 486.42767333984375, + "step": 524 + }, + { + "KL/chosen_KL_mean": -557.0775146484375, + "KL/mean": -778.321044921875, + "KL/rejected_KL_mean": -999.5645751953125, + "KL/std": 509.4033203125, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0012059247819706798, + "fcm_dpo/delta": -0.14199870824813843, + "fcm_dpo/margin": 442.4870300292969, + "fcm_dpo/q_t": 0.38168978691101074, + "grad_norm": 33.15803146362305, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.900769829750061, + "logits/rejected": -0.9381779432296753, + "logps/chosen": -607.9053955078125, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1099.6175537109375, + "loss": 1.0148, + "margin_dpo/margin_mean": 442.48699951171875, + "margin_dpo/margin_std": 537.2896728515625, + "step": 525 + }, + { + "KL/chosen_KL_mean": -607.5784912109375, + "KL/mean": -820.95458984375, + "KL/rejected_KL_mean": -1034.330810546875, + "KL/std": 501.362548828125, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0011866599088534713, + "fcm_dpo/delta": -0.11196226626634598, + "fcm_dpo/margin": 426.75238037109375, + "fcm_dpo/q_t": 0.38492655754089355, + "grad_norm": 30.778478622436523, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.9259464740753174, + "logits/rejected": -0.9302047491073608, + "logps/chosen": -670.7457275390625, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1120.64013671875, + "loss": 1.044, + "margin_dpo/margin_mean": 426.75238037109375, + "margin_dpo/margin_std": 553.6443481445312, + "step": 526 + }, + { + "KL/chosen_KL_mean": -622.232421875, + "KL/mean": -767.9803466796875, + "KL/rejected_KL_mean": -913.728271484375, + "KL/std": 509.15869140625, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0011825388064607978, + "fcm_dpo/delta": 0.05729180574417114, + "fcm_dpo/margin": 291.4958801269531, + "fcm_dpo/q_t": 0.4199332594871521, + "grad_norm": 34.17734146118164, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.0874899625778198, + "logits/rejected": -1.062009572982788, + "logps/chosen": -693.361083984375, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -992.0708618164062, + "loss": 1.135, + "margin_dpo/margin_mean": 291.4958801269531, + "margin_dpo/margin_std": 436.75994873046875, + "step": 527 + }, + { + "KL/chosen_KL_mean": -641.3695068359375, + "KL/mean": -817.936279296875, + "KL/rejected_KL_mean": -994.5032348632812, + "KL/std": 492.47955322265625, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0011843533720821142, + "fcm_dpo/delta": -0.019062494859099388, + "fcm_dpo/margin": 353.1336364746094, + "fcm_dpo/q_t": 0.4085182249546051, + "grad_norm": 47.75295639038086, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -0.9868455529212952, + "logits/rejected": -0.9731815457344055, + "logps/chosen": -709.458984375, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1088.413330078125, + "loss": 1.1184, + "margin_dpo/margin_mean": 353.1336364746094, + "margin_dpo/margin_std": 580.4624633789062, + "step": 528 + }, + { + "KL/chosen_KL_mean": -767.718994140625, + "KL/mean": -888.4722900390625, + "KL/rejected_KL_mean": -1009.2254638671875, + "KL/std": 455.073486328125, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0012000275310128927, + "fcm_dpo/delta": 0.1136150062084198, + "fcm_dpo/margin": 241.50648498535156, + "fcm_dpo/q_t": 0.43466100096702576, + "grad_norm": 44.166587829589844, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.02810537815094, + "logits/rejected": -1.0410661697387695, + "logps/chosen": -823.2939453125, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1098.4345703125, + "loss": 1.2337, + "margin_dpo/margin_mean": 241.50648498535156, + "margin_dpo/margin_std": 577.1032104492188, + "step": 529 + }, + { + "KL/chosen_KL_mean": -653.724365234375, + "KL/mean": -850.6680908203125, + "KL/rejected_KL_mean": -1047.61181640625, + "KL/std": 556.3330078125, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0011986072640866041, + "fcm_dpo/delta": -0.07565183937549591, + "fcm_dpo/margin": 393.88751220703125, + "fcm_dpo/q_t": 0.39879822731018066, + "grad_norm": 52.88919448852539, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.9165508151054382, + "logits/rejected": -0.915734589099884, + "logps/chosen": -701.3258056640625, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1134.8963623046875, + "loss": 1.1197, + "margin_dpo/margin_mean": 393.88751220703125, + "margin_dpo/margin_std": 697.0640869140625, + "step": 530 + }, + { + "KL/chosen_KL_mean": -719.5700073242188, + "KL/mean": -889.66357421875, + "KL/rejected_KL_mean": -1059.7572021484375, + "KL/std": 576.3438720703125, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0011949832551181316, + "fcm_dpo/delta": -0.006902540102601051, + "fcm_dpo/margin": 340.1871032714844, + "fcm_dpo/q_t": 0.4101976752281189, + "grad_norm": 49.53278350830078, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.0011041164398193, + "logits/rejected": -0.9831894636154175, + "logps/chosen": -774.8160400390625, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1130.3631591796875, + "loss": 1.1211, + "margin_dpo/margin_mean": 340.1871337890625, + "margin_dpo/margin_std": 556.56396484375, + "step": 531 + }, + { + "KL/chosen_KL_mean": -712.3829956054688, + "KL/mean": -893.4652099609375, + "KL/rejected_KL_mean": -1074.5474853515625, + "KL/std": 551.279296875, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0011781209614127874, + "fcm_dpo/delta": -0.029086530208587646, + "fcm_dpo/margin": 362.1644592285156, + "fcm_dpo/q_t": 0.4064500331878662, + "grad_norm": 84.89320373535156, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.0040782690048218, + "logits/rejected": -0.9838571548461914, + "logps/chosen": -782.6690063476562, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1161.1387939453125, + "loss": 1.1399, + "margin_dpo/margin_mean": 362.1644592285156, + "margin_dpo/margin_std": 645.1697998046875, + "step": 532 + }, + { + "KL/chosen_KL_mean": -642.6351928710938, + "KL/mean": -773.297119140625, + "KL/rejected_KL_mean": -903.958984375, + "KL/std": 462.89288330078125, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0012072615791112185, + "fcm_dpo/delta": 0.0860050693154335, + "fcm_dpo/margin": 261.32373046875, + "fcm_dpo/q_t": 0.42987143993377686, + "grad_norm": 54.69917678833008, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -0.9916807413101196, + "logits/rejected": -0.9725782871246338, + "logps/chosen": -707.4902954101562, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -980.5469970703125, + "loss": 1.1999, + "margin_dpo/margin_mean": 261.32373046875, + "margin_dpo/margin_std": 552.9267578125, + "step": 533 + }, + { + "KL/chosen_KL_mean": -679.0133666992188, + "KL/mean": -857.49609375, + "KL/rejected_KL_mean": -1035.978759765625, + "KL/std": 523.7025146484375, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.001191169605590403, + "fcm_dpo/delta": -0.027728725224733353, + "fcm_dpo/margin": 356.9653625488281, + "fcm_dpo/q_t": 0.40538090467453003, + "grad_norm": 38.655818939208984, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -0.9712672233581543, + "logits/rejected": -0.9686389565467834, + "logps/chosen": -739.1327514648438, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1114.522216796875, + "loss": 1.1041, + "margin_dpo/margin_mean": 356.9653625488281, + "margin_dpo/margin_std": 538.4713134765625, + "step": 534 + }, + { + "KL/chosen_KL_mean": -559.9479370117188, + "KL/mean": -762.360595703125, + "KL/rejected_KL_mean": -964.7733154296875, + "KL/std": 498.1813659667969, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0011914224596694112, + "fcm_dpo/delta": -0.08686067163944244, + "fcm_dpo/margin": 404.82525634765625, + "fcm_dpo/q_t": 0.3928349018096924, + "grad_norm": 33.77982711791992, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.9069740772247314, + "logits/rejected": -0.9077688455581665, + "logps/chosen": -614.2781982421875, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1061.0809326171875, + "loss": 1.0554, + "margin_dpo/margin_mean": 404.82525634765625, + "margin_dpo/margin_std": 530.0626220703125, + "step": 535 + }, + { + "KL/chosen_KL_mean": -506.74407958984375, + "KL/mean": -752.3280029296875, + "KL/rejected_KL_mean": -997.9119873046875, + "KL/std": 567.429931640625, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0011487666051834822, + "fcm_dpo/delta": -0.17394369840621948, + "fcm_dpo/margin": 491.1679992675781, + "fcm_dpo/q_t": 0.3799936771392822, + "grad_norm": 32.78369903564453, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.8879396915435791, + "logits/rejected": -0.918487548828125, + "logps/chosen": -553.8245849609375, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1087.0098876953125, + "loss": 1.0182, + "margin_dpo/margin_mean": 491.16796875, + "margin_dpo/margin_std": 674.712890625, + "step": 536 + }, + { + "KL/chosen_KL_mean": -521.4559326171875, + "KL/mean": -690.4597778320312, + "KL/rejected_KL_mean": -859.463623046875, + "KL/std": 452.4463195800781, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0011459918459877372, + "fcm_dpo/delta": 0.012487221509218216, + "fcm_dpo/margin": 338.0077209472656, + "fcm_dpo/q_t": 0.41145867109298706, + "grad_norm": 50.913734436035156, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.909199595451355, + "logits/rejected": -0.8892596364021301, + "logps/chosen": -579.2034301757812, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -929.9019775390625, + "loss": 1.1305, + "margin_dpo/margin_mean": 338.0077209472656, + "margin_dpo/margin_std": 546.3211669921875, + "step": 537 + }, + { + "KL/chosen_KL_mean": -655.4517211914062, + "KL/mean": -841.8675537109375, + "KL/rejected_KL_mean": -1028.283447265625, + "KL/std": 488.68878173828125, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0011330239940434694, + "fcm_dpo/delta": -0.02367962710559368, + "fcm_dpo/margin": 372.83172607421875, + "fcm_dpo/q_t": 0.4041064381599426, + "grad_norm": 32.91276168823242, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.919418215751648, + "logits/rejected": -0.9033312797546387, + "logps/chosen": -721.86767578125, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1112.511474609375, + "loss": 1.0828, + "margin_dpo/margin_mean": 372.8316955566406, + "margin_dpo/margin_std": 503.71356201171875, + "step": 538 + }, + { + "KL/chosen_KL_mean": -553.4569091796875, + "KL/mean": -728.4906616210938, + "KL/rejected_KL_mean": -903.5244140625, + "KL/std": 407.0201416015625, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.001137171289883554, + "fcm_dpo/delta": 0.001942652277648449, + "fcm_dpo/margin": 350.06744384765625, + "fcm_dpo/q_t": 0.4085889160633087, + "grad_norm": 35.26192855834961, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.9463549852371216, + "logits/rejected": -0.9348570108413696, + "logps/chosen": -611.9497680664062, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -995.3783569335938, + "loss": 1.0887, + "margin_dpo/margin_mean": 350.06744384765625, + "margin_dpo/margin_std": 448.7100830078125, + "step": 539 + }, + { + "KL/chosen_KL_mean": -558.2035522460938, + "KL/mean": -794.4476318359375, + "KL/rejected_KL_mean": -1030.691650390625, + "KL/std": 524.2433471679688, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0011167166521772742, + "fcm_dpo/delta": -0.13452479243278503, + "fcm_dpo/margin": 472.4881591796875, + "fcm_dpo/q_t": 0.3851798474788666, + "grad_norm": 36.19306182861328, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.9079470634460449, + "logits/rejected": -0.9311023950576782, + "logps/chosen": -621.68603515625, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1147.12158203125, + "loss": 1.0355, + "margin_dpo/margin_mean": 472.4881591796875, + "margin_dpo/margin_std": 620.814453125, + "step": 540 + }, + { + "KL/chosen_KL_mean": -675.7076416015625, + "KL/mean": -797.5556640625, + "KL/rejected_KL_mean": -919.4037475585938, + "KL/std": 470.92388916015625, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0011193943209946156, + "fcm_dpo/delta": 0.1306905746459961, + "fcm_dpo/margin": 243.6960906982422, + "fcm_dpo/q_t": 0.43888312578201294, + "grad_norm": 47.37370681762695, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.0114123821258545, + "logits/rejected": -1.0014359951019287, + "logps/chosen": -753.9879760742188, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1022.6774291992188, + "loss": 1.2286, + "margin_dpo/margin_mean": 243.69610595703125, + "margin_dpo/margin_std": 557.1458740234375, + "step": 541 + }, + { + "KL/chosen_KL_mean": -585.5407104492188, + "KL/mean": -817.8839111328125, + "KL/rejected_KL_mean": -1050.22705078125, + "KL/std": 523.5008544921875, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.0011047889711335301, + "fcm_dpo/delta": -0.12206330895423889, + "fcm_dpo/margin": 464.686279296875, + "fcm_dpo/q_t": 0.38596802949905396, + "grad_norm": 41.06620788574219, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -1.0081770420074463, + "logits/rejected": -1.0378968715667725, + "logps/chosen": -643.0256958007812, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1146.7021484375, + "loss": 1.045, + "margin_dpo/margin_mean": 464.686279296875, + "margin_dpo/margin_std": 617.782958984375, + "step": 542 + }, + { + "KL/chosen_KL_mean": -658.9345703125, + "KL/mean": -823.3148803710938, + "KL/rejected_KL_mean": -987.6951904296875, + "KL/std": 629.5927734375, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0011108315084129572, + "fcm_dpo/delta": 0.03610096499323845, + "fcm_dpo/margin": 328.7606201171875, + "fcm_dpo/q_t": 0.4295271337032318, + "grad_norm": 36.297298431396484, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.9338538646697998, + "logits/rejected": -0.9619553089141846, + "logps/chosen": -719.5521240234375, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1093.294189453125, + "loss": 1.194, + "margin_dpo/margin_mean": 328.7605895996094, + "margin_dpo/margin_std": 739.8629760742188, + "step": 543 + }, + { + "KL/chosen_KL_mean": -672.2225341796875, + "KL/mean": -860.1298828125, + "KL/rejected_KL_mean": -1048.0372314453125, + "KL/std": 510.05084228515625, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.001111794845201075, + "fcm_dpo/delta": -0.018617605790495872, + "fcm_dpo/margin": 375.81475830078125, + "fcm_dpo/q_t": 0.40610426664352417, + "grad_norm": 35.64406204223633, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.9651904106140137, + "logits/rejected": -0.9810581207275391, + "logps/chosen": -731.8648071289062, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1148.991943359375, + "loss": 1.1039, + "margin_dpo/margin_mean": 375.81475830078125, + "margin_dpo/margin_std": 575.191162109375, + "step": 544 + }, + { + "KL/chosen_KL_mean": -619.4461669921875, + "KL/mean": -847.3128051757812, + "KL/rejected_KL_mean": -1075.179443359375, + "KL/std": 508.1304931640625, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0010942972730845213, + "fcm_dpo/delta": -0.10387773811817169, + "fcm_dpo/margin": 455.7331848144531, + "fcm_dpo/q_t": 0.3884674310684204, + "grad_norm": 34.50474548339844, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.8928742408752441, + "logits/rejected": -0.9248363375663757, + "logps/chosen": -687.0947875976562, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1171.08740234375, + "loss": 1.039, + "margin_dpo/margin_mean": 455.733154296875, + "margin_dpo/margin_std": 579.08544921875, + "step": 545 + }, + { + "KL/chosen_KL_mean": -583.528564453125, + "KL/mean": -729.68212890625, + "KL/rejected_KL_mean": -875.835693359375, + "KL/std": 430.79150390625, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.0011012628674507141, + "fcm_dpo/delta": 0.08070985227823257, + "fcm_dpo/margin": 292.3071594238281, + "fcm_dpo/q_t": 0.4241529703140259, + "grad_norm": 40.490291595458984, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.9528399109840393, + "logits/rejected": -0.9377053380012512, + "logps/chosen": -634.2728271484375, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -957.701904296875, + "loss": 1.154, + "margin_dpo/margin_mean": 292.3071594238281, + "margin_dpo/margin_std": 466.27734375, + "step": 546 + }, + { + "KL/chosen_KL_mean": -589.0099487304688, + "KL/mean": -784.4947509765625, + "KL/rejected_KL_mean": -979.979736328125, + "KL/std": 499.3576354980469, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0011031989706680179, + "fcm_dpo/delta": -0.032760508358478546, + "fcm_dpo/margin": 390.9697265625, + "fcm_dpo/q_t": 0.4027066230773926, + "grad_norm": 46.8840446472168, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.9281629323959351, + "logits/rejected": -0.9417147636413574, + "logps/chosen": -662.6976318359375, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1070.7410888671875, + "loss": 1.0837, + "margin_dpo/margin_mean": 390.9697265625, + "margin_dpo/margin_std": 544.32568359375, + "step": 547 + }, + { + "KL/chosen_KL_mean": -616.4374389648438, + "KL/mean": -795.1117553710938, + "KL/rejected_KL_mean": -973.7861328125, + "KL/std": 527.0774536132812, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.001099078101105988, + "fcm_dpo/delta": 0.00753227062523365, + "fcm_dpo/margin": 357.3486022949219, + "fcm_dpo/q_t": 0.41554519534111023, + "grad_norm": 29.250102996826172, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.9849483966827393, + "logits/rejected": -0.9840346574783325, + "logps/chosen": -681.6837768554688, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1067.9041748046875, + "loss": 1.1189, + "margin_dpo/margin_mean": 357.3486328125, + "margin_dpo/margin_std": 587.6913452148438, + "step": 548 + }, + { + "KL/chosen_KL_mean": -654.0398559570312, + "KL/mean": -790.404296875, + "KL/rejected_KL_mean": -926.768798828125, + "KL/std": 449.04010009765625, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0011215468402951956, + "fcm_dpo/delta": 0.0962221547961235, + "fcm_dpo/margin": 272.7288818359375, + "fcm_dpo/q_t": 0.43191951513290405, + "grad_norm": 65.5692367553711, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.9725868701934814, + "logits/rejected": -0.9614546298980713, + "logps/chosen": -703.252197265625, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1000.6790771484375, + "loss": 1.1888, + "margin_dpo/margin_mean": 272.7288818359375, + "margin_dpo/margin_std": 533.6253051757812, + "step": 549 + }, + { + "KL/chosen_KL_mean": -650.1110229492188, + "KL/mean": -808.8392333984375, + "KL/rejected_KL_mean": -967.5673828125, + "KL/std": 486.4993896484375, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.001128158182837069, + "fcm_dpo/delta": 0.043436355888843536, + "fcm_dpo/margin": 317.4564208984375, + "fcm_dpo/q_t": 0.41883769631385803, + "grad_norm": 36.942352294921875, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.9538976550102234, + "logits/rejected": -0.9776439666748047, + "logps/chosen": -706.91796875, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1062.6932373046875, + "loss": 1.1353, + "margin_dpo/margin_mean": 317.4563903808594, + "margin_dpo/margin_std": 509.97784423828125, + "step": 550 + }, + { + "KL/chosen_KL_mean": -563.455078125, + "KL/mean": -827.8677978515625, + "KL/rejected_KL_mean": -1092.2806396484375, + "KL/std": 518.2503051757812, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.001097839791327715, + "fcm_dpo/delta": -0.19212040305137634, + "fcm_dpo/margin": 528.8255615234375, + "fcm_dpo/q_t": 0.36717739701271057, + "grad_norm": 53.38359451293945, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.9228482842445374, + "logits/rejected": -0.9759207963943481, + "logps/chosen": -622.5614013671875, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1203.953369140625, + "loss": 0.9597, + "margin_dpo/margin_mean": 528.8256225585938, + "margin_dpo/margin_std": 517.7742309570312, + "step": 551 + }, + { + "KL/chosen_KL_mean": -553.8248291015625, + "KL/mean": -846.4808349609375, + "KL/rejected_KL_mean": -1139.13671875, + "KL/std": 600.181640625, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0010433063143864274, + "fcm_dpo/delta": -0.2280970811843872, + "fcm_dpo/margin": 585.3118896484375, + "fcm_dpo/q_t": 0.3672313094139099, + "grad_norm": 52.089378356933594, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.9867992401123047, + "logits/rejected": -1.0266941785812378, + "logps/chosen": -616.179443359375, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1243.6988525390625, + "loss": 0.9737, + "margin_dpo/margin_mean": 585.3118896484375, + "margin_dpo/margin_std": 678.860107421875, + "step": 552 + }, + { + "KL/chosen_KL_mean": -626.0472412109375, + "KL/mean": -779.4678955078125, + "KL/rejected_KL_mean": -932.8885498046875, + "KL/std": 495.7182922363281, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0010498525807633996, + "fcm_dpo/delta": 0.08051052689552307, + "fcm_dpo/margin": 306.8413391113281, + "fcm_dpo/q_t": 0.4286406636238098, + "grad_norm": 27.60106086730957, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.9779776334762573, + "logits/rejected": -0.9744440317153931, + "logps/chosen": -694.3060302734375, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1030.9857177734375, + "loss": 1.1633, + "margin_dpo/margin_mean": 306.84136962890625, + "margin_dpo/margin_std": 538.6355590820312, + "step": 553 + }, + { + "KL/chosen_KL_mean": -649.279052734375, + "KL/mean": -858.0142822265625, + "KL/rejected_KL_mean": -1066.74951171875, + "KL/std": 546.52490234375, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.00105759606231004, + "fcm_dpo/delta": -0.04407143592834473, + "fcm_dpo/margin": 417.47039794921875, + "fcm_dpo/q_t": 0.40395021438598633, + "grad_norm": 92.40606689453125, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.9501082301139832, + "logits/rejected": -0.9274972081184387, + "logps/chosen": -717.2267456054688, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1156.5322265625, + "loss": 1.1185, + "margin_dpo/margin_mean": 417.470458984375, + "margin_dpo/margin_std": 696.6802978515625, + "step": 554 + }, + { + "KL/chosen_KL_mean": -663.811279296875, + "KL/mean": -905.8134155273438, + "KL/rejected_KL_mean": -1147.8155517578125, + "KL/std": 580.5455932617188, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0010336714331060648, + "fcm_dpo/delta": -0.10545919835567474, + "fcm_dpo/margin": 484.0042724609375, + "fcm_dpo/q_t": 0.3939095139503479, + "grad_norm": 37.96812438964844, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.0255415439605713, + "logits/rejected": -1.0774145126342773, + "logps/chosen": -717.1417236328125, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1256.294921875, + "loss": 1.0777, + "margin_dpo/margin_mean": 484.0042724609375, + "margin_dpo/margin_std": 753.3184814453125, + "step": 555 + }, + { + "KL/chosen_KL_mean": -636.8773193359375, + "KL/mean": -779.0126953125, + "KL/rejected_KL_mean": -921.1480712890625, + "KL/std": 440.1156311035156, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0010426306398585439, + "fcm_dpo/delta": 0.1067805290222168, + "fcm_dpo/margin": 284.2707824707031, + "fcm_dpo/q_t": 0.4317725598812103, + "grad_norm": 29.631032943725586, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.980458676815033, + "logits/rejected": -0.976431131362915, + "logps/chosen": -695.521728515625, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1022.4884643554688, + "loss": 1.1627, + "margin_dpo/margin_mean": 284.2707824707031, + "margin_dpo/margin_std": 445.5572204589844, + "step": 556 + }, + { + "KL/chosen_KL_mean": -674.95654296875, + "KL/mean": -900.3138427734375, + "KL/rejected_KL_mean": -1125.671142578125, + "KL/std": 609.170654296875, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0010350887896493077, + "fcm_dpo/delta": -0.07005338370800018, + "fcm_dpo/margin": 450.7147216796875, + "fcm_dpo/q_t": 0.3982085585594177, + "grad_norm": 59.1350212097168, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.0268800258636475, + "logits/rejected": -1.0635356903076172, + "logps/chosen": -742.797119140625, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1235.61083984375, + "loss": 1.0881, + "margin_dpo/margin_mean": 450.7147216796875, + "margin_dpo/margin_std": 688.540771484375, + "step": 557 + }, + { + "KL/chosen_KL_mean": -615.1371459960938, + "KL/mean": -908.5450439453125, + "KL/rejected_KL_mean": -1201.952880859375, + "KL/std": 567.5309448242188, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0010039007756859064, + "fcm_dpo/delta": -0.2008526623249054, + "fcm_dpo/margin": 586.8157958984375, + "fcm_dpo/q_t": 0.3645177483558655, + "grad_norm": 30.52578353881836, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.059295415878296, + "logits/rejected": -1.083045244216919, + "logps/chosen": -677.50537109375, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1304.114013671875, + "loss": 0.9728, + "margin_dpo/margin_mean": 586.8157958984375, + "margin_dpo/margin_std": 630.1806640625, + "step": 558 + }, + { + "KL/chosen_KL_mean": -708.15625, + "KL/mean": -953.3082885742188, + "KL/rejected_KL_mean": -1198.4603271484375, + "KL/std": 579.6380004882812, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0009818286634981632, + "fcm_dpo/delta": -0.08543342351913452, + "fcm_dpo/margin": 490.30401611328125, + "fcm_dpo/q_t": 0.39213061332702637, + "grad_norm": 29.082914352416992, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.1655193567276, + "logits/rejected": -1.1723744869232178, + "logps/chosen": -768.9085693359375, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1291.902587890625, + "loss": 1.0486, + "margin_dpo/margin_mean": 490.30401611328125, + "margin_dpo/margin_std": 633.2149658203125, + "step": 559 + }, + { + "KL/chosen_KL_mean": -635.3558349609375, + "KL/mean": -815.1640625, + "KL/rejected_KL_mean": -994.9723510742188, + "KL/std": 509.7801818847656, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0009786419104784727, + "fcm_dpo/delta": 0.049565140157938004, + "fcm_dpo/margin": 359.6164855957031, + "fcm_dpo/q_t": 0.419849157333374, + "grad_norm": 30.310014724731445, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.9133145809173584, + "logits/rejected": -0.8977816104888916, + "logps/chosen": -693.4596557617188, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1074.963623046875, + "loss": 1.1364, + "margin_dpo/margin_mean": 359.6164855957031, + "margin_dpo/margin_std": 555.07275390625, + "step": 560 + }, + { + "KL/chosen_KL_mean": -743.4735717773438, + "KL/mean": -892.7384033203125, + "KL/rejected_KL_mean": -1042.003173828125, + "KL/std": 509.44683837890625, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0010041221976280212, + "fcm_dpo/delta": 0.10286542773246765, + "fcm_dpo/margin": 298.52960205078125, + "fcm_dpo/q_t": 0.4302954077720642, + "grad_norm": 42.739471435546875, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.071549415588379, + "logits/rejected": -1.0916610956192017, + "logps/chosen": -794.386474609375, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1141.07177734375, + "loss": 1.2016, + "margin_dpo/margin_mean": 298.52960205078125, + "margin_dpo/margin_std": 629.797119140625, + "step": 561 + }, + { + "KL/chosen_KL_mean": -689.78515625, + "KL/mean": -897.9728393554688, + "KL/rejected_KL_mean": -1106.1605224609375, + "KL/std": 560.4344482421875, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.000997741473838687, + "fcm_dpo/delta": -0.01698533073067665, + "fcm_dpo/margin": 416.3754577636719, + "fcm_dpo/q_t": 0.4067772626876831, + "grad_norm": 46.461944580078125, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.034153938293457, + "logits/rejected": -1.048844337463379, + "logps/chosen": -749.24951171875, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1202.7032470703125, + "loss": 1.0982, + "margin_dpo/margin_mean": 416.37542724609375, + "margin_dpo/margin_std": 609.9890747070312, + "step": 562 + }, + { + "KL/chosen_KL_mean": -763.398681640625, + "KL/mean": -915.3447875976562, + "KL/rejected_KL_mean": -1067.2908935546875, + "KL/std": 605.0965576171875, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.001025867648422718, + "fcm_dpo/delta": 0.08850602805614471, + "fcm_dpo/margin": 303.89227294921875, + "fcm_dpo/q_t": 0.42389655113220215, + "grad_norm": 51.39828872680664, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.0280288457870483, + "logits/rejected": -1.0322433710098267, + "logps/chosen": -824.0068359375, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1161.858642578125, + "loss": 1.2334, + "margin_dpo/margin_mean": 303.89227294921875, + "margin_dpo/margin_std": 717.966552734375, + "step": 563 + }, + { + "KL/chosen_KL_mean": -679.8832397460938, + "KL/mean": -879.3602905273438, + "KL/rejected_KL_mean": -1078.83740234375, + "KL/std": 527.3673706054688, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0010222452692687511, + "fcm_dpo/delta": -0.008269475772976875, + "fcm_dpo/margin": 398.9542236328125, + "fcm_dpo/q_t": 0.40674155950546265, + "grad_norm": 43.34153747558594, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.1044118404388428, + "logits/rejected": -1.100728154182434, + "logps/chosen": -754.151611328125, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1172.07568359375, + "loss": 1.0825, + "margin_dpo/margin_mean": 398.9542236328125, + "margin_dpo/margin_std": 504.2801513671875, + "step": 564 + }, + { + "KL/chosen_KL_mean": -715.0086669921875, + "KL/mean": -928.3496704101562, + "KL/rejected_KL_mean": -1141.690673828125, + "KL/std": 611.9755249023438, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0010138817597180605, + "fcm_dpo/delta": -0.03408730775117874, + "fcm_dpo/margin": 426.68194580078125, + "fcm_dpo/q_t": 0.4070592224597931, + "grad_norm": 53.82391357421875, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.052903175354004, + "logits/rejected": -1.0597248077392578, + "logps/chosen": -784.0286254882812, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1227.469482421875, + "loss": 1.1219, + "margin_dpo/margin_mean": 426.68194580078125, + "margin_dpo/margin_std": 732.296630859375, + "step": 565 + }, + { + "KL/chosen_KL_mean": -688.0936279296875, + "KL/mean": -924.2864379882812, + "KL/rejected_KL_mean": -1160.479248046875, + "KL/std": 630.7330322265625, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0010061727371066809, + "fcm_dpo/delta": -0.07926531136035919, + "fcm_dpo/margin": 472.3856201171875, + "fcm_dpo/q_t": 0.39555519819259644, + "grad_norm": 37.58675003051758, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.0540146827697754, + "logits/rejected": -1.083146572113037, + "logps/chosen": -754.638916015625, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1264.3486328125, + "loss": 1.0919, + "margin_dpo/margin_mean": 472.3856201171875, + "margin_dpo/margin_std": 729.4598999023438, + "step": 566 + }, + { + "KL/chosen_KL_mean": -652.9796142578125, + "KL/mean": -830.4371948242188, + "KL/rejected_KL_mean": -1007.894775390625, + "KL/std": 433.50579833984375, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0010030196281149983, + "fcm_dpo/delta": 0.04564574360847473, + "fcm_dpo/margin": 354.91522216796875, + "fcm_dpo/q_t": 0.41683584451675415, + "grad_norm": 37.2297248840332, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.1648993492126465, + "logits/rejected": -1.179152488708496, + "logps/chosen": -705.837890625, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1093.2657470703125, + "loss": 1.1158, + "margin_dpo/margin_mean": 354.9151611328125, + "margin_dpo/margin_std": 479.8267517089844, + "step": 567 + }, + { + "KL/chosen_KL_mean": -639.865966796875, + "KL/mean": -881.840576171875, + "KL/rejected_KL_mean": -1123.815185546875, + "KL/std": 509.06475830078125, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0009923405013978481, + "fcm_dpo/delta": -0.08435662090778351, + "fcm_dpo/margin": 483.9492492675781, + "fcm_dpo/q_t": 0.38865938782691956, + "grad_norm": 35.289207458496094, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.015796422958374, + "logits/rejected": -1.0558149814605713, + "logps/chosen": -685.058349609375, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1212.90771484375, + "loss": 1.0285, + "margin_dpo/margin_mean": 483.94927978515625, + "margin_dpo/margin_std": 540.4320068359375, + "step": 568 + }, + { + "KL/chosen_KL_mean": -730.6180419921875, + "KL/mean": -924.656982421875, + "KL/rejected_KL_mean": -1118.69580078125, + "KL/std": 653.961669921875, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0009969357633963227, + "fcm_dpo/delta": 0.012732669711112976, + "fcm_dpo/margin": 388.0777587890625, + "fcm_dpo/q_t": 0.41255781054496765, + "grad_norm": 48.96441650390625, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.0476577281951904, + "logits/rejected": -1.0544722080230713, + "logps/chosen": -794.338623046875, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1197.799072265625, + "loss": 1.1502, + "margin_dpo/margin_mean": 388.0777587890625, + "margin_dpo/margin_std": 688.9151611328125, + "step": 569 + }, + { + "KL/chosen_KL_mean": -696.5452880859375, + "KL/mean": -878.316650390625, + "KL/rejected_KL_mean": -1060.087890625, + "KL/std": 534.93408203125, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0009956157300621271, + "fcm_dpo/delta": 0.03949379175901413, + "fcm_dpo/margin": 363.54266357421875, + "fcm_dpo/q_t": 0.4193030595779419, + "grad_norm": 34.15614318847656, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.1255332231521606, + "logits/rejected": -1.1205517053604126, + "logps/chosen": -758.1598510742188, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1142.2298583984375, + "loss": 1.1367, + "margin_dpo/margin_mean": 363.5426940917969, + "margin_dpo/margin_std": 587.5882568359375, + "step": 570 + }, + { + "KL/chosen_KL_mean": -692.908447265625, + "KL/mean": -913.161865234375, + "KL/rejected_KL_mean": -1133.415283203125, + "KL/std": 538.54736328125, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.000990982400253415, + "fcm_dpo/delta": -0.03841770812869072, + "fcm_dpo/margin": 440.5067138671875, + "fcm_dpo/q_t": 0.40229225158691406, + "grad_norm": 46.066192626953125, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -0.9920418858528137, + "logits/rejected": -1.007279634475708, + "logps/chosen": -745.9625244140625, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1224.7520751953125, + "loss": 1.0977, + "margin_dpo/margin_mean": 440.5067138671875, + "margin_dpo/margin_std": 668.992431640625, + "step": 571 + }, + { + "KL/chosen_KL_mean": -740.664794921875, + "KL/mean": -971.2442626953125, + "KL/rejected_KL_mean": -1201.82373046875, + "KL/std": 629.68310546875, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0009805567096918821, + "fcm_dpo/delta": -0.05499058961868286, + "fcm_dpo/margin": 461.15899658203125, + "fcm_dpo/q_t": 0.4029189348220825, + "grad_norm": 32.28114700317383, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.0524401664733887, + "logits/rejected": -1.0868430137634277, + "logps/chosen": -789.1240234375, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1285.380859375, + "loss": 1.1047, + "margin_dpo/margin_mean": 461.15899658203125, + "margin_dpo/margin_std": 759.63232421875, + "step": 572 + }, + { + "KL/chosen_KL_mean": -700.3189697265625, + "KL/mean": -896.9549560546875, + "KL/rejected_KL_mean": -1093.5909423828125, + "KL/std": 553.4898681640625, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.000984402373433113, + "fcm_dpo/delta": 0.013283109292387962, + "fcm_dpo/margin": 393.27197265625, + "fcm_dpo/q_t": 0.41296589374542236, + "grad_norm": 27.198640823364258, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.0818779468536377, + "logits/rejected": -1.0913732051849365, + "logps/chosen": -751.941650390625, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1178.916015625, + "loss": 1.1026, + "margin_dpo/margin_mean": 393.27197265625, + "margin_dpo/margin_std": 554.1753540039062, + "step": 573 + }, + { + "KL/chosen_KL_mean": -643.3096923828125, + "KL/mean": -856.635009765625, + "KL/rejected_KL_mean": -1069.96044921875, + "KL/std": 503.39013671875, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0009823552099987864, + "fcm_dpo/delta": -0.020006868988275528, + "fcm_dpo/margin": 426.65069580078125, + "fcm_dpo/q_t": 0.40471595525741577, + "grad_norm": 42.36172103881836, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.0936473608016968, + "logits/rejected": -1.1251025199890137, + "logps/chosen": -694.3541870117188, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1162.766845703125, + "loss": 1.087, + "margin_dpo/margin_mean": 426.65069580078125, + "margin_dpo/margin_std": 583.6290283203125, + "step": 574 + }, + { + "KL/chosen_KL_mean": -704.0703125, + "KL/mean": -902.9998779296875, + "KL/rejected_KL_mean": -1101.929443359375, + "KL/std": 509.685791015625, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0009733311599120498, + "fcm_dpo/delta": 0.012073688209056854, + "fcm_dpo/margin": 397.85931396484375, + "fcm_dpo/q_t": 0.4121411442756653, + "grad_norm": 38.166893005371094, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.9443089962005615, + "logits/rejected": -0.9500855207443237, + "logps/chosen": -775.8604125976562, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1197.315673828125, + "loss": 1.1382, + "margin_dpo/margin_mean": 397.8593444824219, + "margin_dpo/margin_std": 660.2369384765625, + "step": 575 + }, + { + "KL/chosen_KL_mean": -697.45458984375, + "KL/mean": -845.8167724609375, + "KL/rejected_KL_mean": -994.1790161132812, + "KL/std": 480.55767822265625, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0009943852201104164, + "fcm_dpo/delta": 0.10826772451400757, + "fcm_dpo/margin": 296.72442626953125, + "fcm_dpo/q_t": 0.43276447057724, + "grad_norm": 35.01775360107422, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.0605216026306152, + "logits/rejected": -1.087602972984314, + "logps/chosen": -751.717529296875, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1094.933349609375, + "loss": 1.1866, + "margin_dpo/margin_mean": 296.7244567871094, + "margin_dpo/margin_std": 555.0670776367188, + "step": 576 + }, + { + "KL/chosen_KL_mean": -612.8695068359375, + "KL/mean": -812.4486083984375, + "KL/rejected_KL_mean": -1012.0276489257812, + "KL/std": 539.10009765625, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.001001616707071662, + "fcm_dpo/delta": 0.00015571992844343185, + "fcm_dpo/margin": 399.15814208984375, + "fcm_dpo/q_t": 0.41023534536361694, + "grad_norm": 29.19010353088379, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.0525541305541992, + "logits/rejected": -1.0463950634002686, + "logps/chosen": -674.7792358398438, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1096.098388671875, + "loss": 1.1169, + "margin_dpo/margin_mean": 399.15814208984375, + "margin_dpo/margin_std": 595.0013427734375, + "step": 577 + }, + { + "KL/chosen_KL_mean": -592.90283203125, + "KL/mean": -809.75390625, + "KL/rejected_KL_mean": -1026.6051025390625, + "KL/std": 523.7479858398438, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.0009930215310305357, + "fcm_dpo/delta": -0.03280441835522652, + "fcm_dpo/margin": 433.7021484375, + "fcm_dpo/q_t": 0.4012163281440735, + "grad_norm": 48.96551513671875, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.9256601333618164, + "logits/rejected": -0.9595029354095459, + "logps/chosen": -642.16650390625, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1110.041259765625, + "loss": 1.0663, + "margin_dpo/margin_mean": 433.7021484375, + "margin_dpo/margin_std": 524.7673950195312, + "step": 578 + }, + { + "KL/chosen_KL_mean": -689.10107421875, + "KL/mean": -823.37548828125, + "KL/rejected_KL_mean": -957.6500244140625, + "KL/std": 519.365234375, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0010139856021851301, + "fcm_dpo/delta": 0.1314682960510254, + "fcm_dpo/margin": 268.54888916015625, + "fcm_dpo/q_t": 0.43766841292381287, + "grad_norm": 56.03416442871094, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.9776114821434021, + "logits/rejected": -0.9796488285064697, + "logps/chosen": -747.9869384765625, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1052.4376220703125, + "loss": 1.2106, + "margin_dpo/margin_mean": 268.54888916015625, + "margin_dpo/margin_std": 562.98291015625, + "step": 579 + }, + { + "KL/chosen_KL_mean": -497.3514404296875, + "KL/mean": -740.3182983398438, + "KL/rejected_KL_mean": -983.2850952148438, + "KL/std": 575.2335205078125, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0010052667930722237, + "fcm_dpo/delta": -0.09367062896490097, + "fcm_dpo/margin": 485.9337158203125, + "fcm_dpo/q_t": 0.39301222562789917, + "grad_norm": 35.55183029174805, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8926676511764526, + "logits/rejected": -0.9264934062957764, + "logps/chosen": -546.0582885742188, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1065.04345703125, + "loss": 1.049, + "margin_dpo/margin_mean": 485.9337463378906, + "margin_dpo/margin_std": 657.4573974609375, + "step": 580 + }, + { + "KL/chosen_KL_mean": -680.5172729492188, + "KL/mean": -852.0799560546875, + "KL/rejected_KL_mean": -1023.642578125, + "KL/std": 547.046630859375, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.0010138069046661258, + "fcm_dpo/delta": 0.053889188915491104, + "fcm_dpo/margin": 343.1253967285156, + "fcm_dpo/q_t": 0.42307010293006897, + "grad_norm": 40.41847229003906, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.0801353454589844, + "logits/rejected": -1.0921562910079956, + "logps/chosen": -742.7396240234375, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1115.3782958984375, + "loss": 1.1525, + "margin_dpo/margin_mean": 343.1253967285156, + "margin_dpo/margin_std": 606.5958862304688, + "step": 581 + }, + { + "KL/chosen_KL_mean": -609.591552734375, + "KL/mean": -800.7734985351562, + "KL/rejected_KL_mean": -991.9555053710938, + "KL/std": 482.5928955078125, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0010254253866150975, + "fcm_dpo/delta": 0.006468296051025391, + "fcm_dpo/margin": 382.3638916015625, + "fcm_dpo/q_t": 0.40925300121307373, + "grad_norm": 33.561561584472656, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -1.0454349517822266, + "logits/rejected": -1.067221760749817, + "logps/chosen": -667.8201904296875, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1102.025146484375, + "loss": 1.1075, + "margin_dpo/margin_mean": 382.3638916015625, + "margin_dpo/margin_std": 520.2689208984375, + "step": 582 + }, + { + "KL/chosen_KL_mean": -700.315185546875, + "KL/mean": -821.8018798828125, + "KL/rejected_KL_mean": -943.2886962890625, + "KL/std": 529.8026733398438, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0010306700132787228, + "fcm_dpo/delta": 0.1531425267457962, + "fcm_dpo/margin": 242.97352600097656, + "fcm_dpo/q_t": 0.44552117586135864, + "grad_norm": 68.53507232666016, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.966422438621521, + "logits/rejected": -0.9622797966003418, + "logps/chosen": -757.1780395507812, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1031.692626953125, + "loss": 1.2624, + "margin_dpo/margin_mean": 242.9735107421875, + "margin_dpo/margin_std": 639.3092041015625, + "step": 583 + }, + { + "KL/chosen_KL_mean": -590.7367553710938, + "KL/mean": -833.6219482421875, + "KL/rejected_KL_mean": -1076.507080078125, + "KL/std": 536.196533203125, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0010323836468160152, + "fcm_dpo/delta": -0.10678394883871078, + "fcm_dpo/margin": 485.7703552246094, + "fcm_dpo/q_t": 0.3871385157108307, + "grad_norm": 31.813514709472656, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.0203688144683838, + "logits/rejected": -1.0354740619659424, + "logps/chosen": -647.637451171875, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1174.14306640625, + "loss": 1.0256, + "margin_dpo/margin_mean": 485.7703857421875, + "margin_dpo/margin_std": 567.4168701171875, + "step": 584 + }, + { + "KL/chosen_KL_mean": -692.355224609375, + "KL/mean": -916.0726928710938, + "KL/rejected_KL_mean": -1139.7901611328125, + "KL/std": 614.073486328125, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0010137277422472835, + "fcm_dpo/delta": -0.056564219295978546, + "fcm_dpo/margin": 447.4348449707031, + "fcm_dpo/q_t": 0.3991103768348694, + "grad_norm": 34.30695724487305, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.0806384086608887, + "logits/rejected": -1.1092705726623535, + "logps/chosen": -753.3292236328125, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1223.959716796875, + "loss": 1.108, + "margin_dpo/margin_mean": 447.4348449707031, + "margin_dpo/margin_std": 737.2296142578125, + "step": 585 + }, + { + "KL/chosen_KL_mean": -717.5431518554688, + "KL/mean": -868.3179321289062, + "KL/rejected_KL_mean": -1019.0927734375, + "KL/std": 531.0343017578125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.001026843674480915, + "fcm_dpo/delta": 0.09334185719490051, + "fcm_dpo/margin": 301.54962158203125, + "fcm_dpo/q_t": 0.42962875962257385, + "grad_norm": 34.41300582885742, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.056232213973999, + "logits/rejected": -1.079580545425415, + "logps/chosen": -774.3987426757812, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1110.8953857421875, + "loss": 1.185, + "margin_dpo/margin_mean": 301.54962158203125, + "margin_dpo/margin_std": 582.9779052734375, + "step": 586 + }, + { + "KL/chosen_KL_mean": -523.61328125, + "KL/mean": -725.7652587890625, + "KL/rejected_KL_mean": -927.917236328125, + "KL/std": 560.800048828125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0010330535005778074, + "fcm_dpo/delta": -0.01851090043783188, + "fcm_dpo/margin": 404.3040466308594, + "fcm_dpo/q_t": 0.40489882230758667, + "grad_norm": 35.46411895751953, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.0895702838897705, + "logits/rejected": -1.1170220375061035, + "logps/chosen": -568.3048706054688, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1010.5411376953125, + "loss": 1.0936, + "margin_dpo/margin_mean": 404.30401611328125, + "margin_dpo/margin_std": 578.6407470703125, + "step": 587 + }, + { + "KL/chosen_KL_mean": -680.740234375, + "KL/mean": -855.3609619140625, + "KL/rejected_KL_mean": -1029.981689453125, + "KL/std": 488.6011657714844, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0010310993529856205, + "fcm_dpo/delta": 0.041131071746349335, + "fcm_dpo/margin": 349.24139404296875, + "fcm_dpo/q_t": 0.4170614182949066, + "grad_norm": 27.398632049560547, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.0426102876663208, + "logits/rejected": -1.0672667026519775, + "logps/chosen": -731.0352172851562, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1137.3515625, + "loss": 1.1273, + "margin_dpo/margin_mean": 349.2414245605469, + "margin_dpo/margin_std": 515.8612060546875, + "step": 588 + }, + { + "KL/chosen_KL_mean": -664.336669921875, + "KL/mean": -895.1517944335938, + "KL/rejected_KL_mean": -1125.967041015625, + "KL/std": 539.49755859375, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0010269335471093655, + "fcm_dpo/delta": -0.07770368456840515, + "fcm_dpo/margin": 461.63037109375, + "fcm_dpo/q_t": 0.3921104669570923, + "grad_norm": 32.468257904052734, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -0.9668374061584473, + "logits/rejected": -0.9841240644454956, + "logps/chosen": -724.2666015625, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1237.622314453125, + "loss": 1.0441, + "margin_dpo/margin_mean": 461.63037109375, + "margin_dpo/margin_std": 566.55224609375, + "step": 589 + }, + { + "KL/chosen_KL_mean": -581.8033447265625, + "KL/mean": -807.3217163085938, + "KL/rejected_KL_mean": -1032.840087890625, + "KL/std": 528.1049194335938, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0010122577659785748, + "fcm_dpo/delta": -0.05933629721403122, + "fcm_dpo/margin": 451.03680419921875, + "fcm_dpo/q_t": 0.39698195457458496, + "grad_norm": 23.684188842773438, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.9741607308387756, + "logits/rejected": -1.0013905763626099, + "logps/chosen": -637.6131591796875, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1138.90283203125, + "loss": 1.0576, + "margin_dpo/margin_mean": 451.0367736816406, + "margin_dpo/margin_std": 575.8516845703125, + "step": 590 + }, + { + "KL/chosen_KL_mean": -631.1538696289062, + "KL/mean": -835.04345703125, + "KL/rejected_KL_mean": -1038.93310546875, + "KL/std": 498.9157409667969, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0010072027798742056, + "fcm_dpo/delta": -0.011214686557650566, + "fcm_dpo/margin": 407.7791442871094, + "fcm_dpo/q_t": 0.4051218032836914, + "grad_norm": 37.07765579223633, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.0779905319213867, + "logits/rejected": -1.0750203132629395, + "logps/chosen": -687.39453125, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1122.71923828125, + "loss": 1.0918, + "margin_dpo/margin_mean": 407.7791442871094, + "margin_dpo/margin_std": 560.2435302734375, + "step": 591 + }, + { + "KL/chosen_KL_mean": -661.8003540039062, + "KL/mean": -846.705322265625, + "KL/rejected_KL_mean": -1031.610107421875, + "KL/std": 514.6201171875, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0010113526368513703, + "fcm_dpo/delta": 0.027010049670934677, + "fcm_dpo/margin": 369.809814453125, + "fcm_dpo/q_t": 0.41588273644447327, + "grad_norm": 35.1633186340332, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.025916337966919, + "logits/rejected": -1.029652714729309, + "logps/chosen": -709.7406005859375, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1107.343017578125, + "loss": 1.1672, + "margin_dpo/margin_mean": 369.809814453125, + "margin_dpo/margin_std": 699.1980590820312, + "step": 592 + }, + { + "KL/chosen_KL_mean": -643.7540283203125, + "KL/mean": -792.77880859375, + "KL/rejected_KL_mean": -941.8035888671875, + "KL/std": 564.8258056640625, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0010245954617857933, + "fcm_dpo/delta": 0.09769494831562042, + "fcm_dpo/margin": 298.04949951171875, + "fcm_dpo/q_t": 0.43263185024261475, + "grad_norm": 47.203948974609375, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.917668342590332, + "logits/rejected": -0.9115473031997681, + "logps/chosen": -692.44482421875, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1002.7115478515625, + "loss": 1.2008, + "margin_dpo/margin_mean": 298.04949951171875, + "margin_dpo/margin_std": 622.1203002929688, + "step": 593 + }, + { + "KL/chosen_KL_mean": -613.9559936523438, + "KL/mean": -802.759033203125, + "KL/rejected_KL_mean": -991.5620727539062, + "KL/std": 546.4432983398438, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.001041522715240717, + "fcm_dpo/delta": 0.005921036005020142, + "fcm_dpo/margin": 377.60601806640625, + "fcm_dpo/q_t": 0.4113161265850067, + "grad_norm": 42.25575256347656, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.0149271488189697, + "logits/rejected": -1.0100760459899902, + "logps/chosen": -668.890869140625, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1077.6617431640625, + "loss": 1.1206, + "margin_dpo/margin_mean": 377.60601806640625, + "margin_dpo/margin_std": 577.789306640625, + "step": 594 + }, + { + "KL/chosen_KL_mean": -579.5828247070312, + "KL/mean": -791.528076171875, + "KL/rejected_KL_mean": -1003.4732666015625, + "KL/std": 520.9078369140625, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.00102597801014781, + "fcm_dpo/delta": -0.03674547001719475, + "fcm_dpo/margin": 423.89044189453125, + "fcm_dpo/q_t": 0.4040411710739136, + "grad_norm": 39.86050033569336, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.9882222414016724, + "logits/rejected": -1.0088510513305664, + "logps/chosen": -629.0032348632812, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1084.1005859375, + "loss": 1.0769, + "margin_dpo/margin_mean": 423.89044189453125, + "margin_dpo/margin_std": 582.7005615234375, + "step": 595 + }, + { + "KL/chosen_KL_mean": -668.828857421875, + "KL/mean": -818.2108154296875, + "KL/rejected_KL_mean": -967.5927124023438, + "KL/std": 511.14495849609375, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0010419124737381935, + "fcm_dpo/delta": 0.09146802127361298, + "fcm_dpo/margin": 298.7638854980469, + "fcm_dpo/q_t": 0.43200355768203735, + "grad_norm": 59.21998977661133, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.988633394241333, + "logits/rejected": -0.9593174457550049, + "logps/chosen": -728.62060546875, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1047.683837890625, + "loss": 1.1918, + "margin_dpo/margin_mean": 298.76385498046875, + "margin_dpo/margin_std": 612.1651611328125, + "step": 596 + }, + { + "KL/chosen_KL_mean": -627.0169677734375, + "KL/mean": -878.0928955078125, + "KL/rejected_KL_mean": -1129.1688232421875, + "KL/std": 584.7470092773438, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0010169618763029575, + "fcm_dpo/delta": -0.11996593326330185, + "fcm_dpo/margin": 502.1518859863281, + "fcm_dpo/q_t": 0.3871939480304718, + "grad_norm": 32.3339729309082, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.893456757068634, + "logits/rejected": -0.9415876865386963, + "logps/chosen": -684.2777099609375, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1229.862548828125, + "loss": 1.0421, + "margin_dpo/margin_mean": 502.1518859863281, + "margin_dpo/margin_std": 639.797119140625, + "step": 597 + }, + { + "KL/chosen_KL_mean": -643.2660522460938, + "KL/mean": -848.2061767578125, + "KL/rejected_KL_mean": -1053.146240234375, + "KL/std": 527.8873291015625, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0010101067600771785, + "fcm_dpo/delta": -0.016199514269828796, + "fcm_dpo/margin": 409.88031005859375, + "fcm_dpo/q_t": 0.4049733281135559, + "grad_norm": 41.460880279541016, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.0348477363586426, + "logits/rejected": -1.0516211986541748, + "logps/chosen": -695.7845458984375, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1142.590087890625, + "loss": 1.0968, + "margin_dpo/margin_mean": 409.88031005859375, + "margin_dpo/margin_std": 566.6337890625, + "step": 598 + }, + { + "KL/chosen_KL_mean": -658.955810546875, + "KL/mean": -821.4459228515625, + "KL/rejected_KL_mean": -983.9359130859375, + "KL/std": 497.92340087890625, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.001019275514408946, + "fcm_dpo/delta": 0.07008726894855499, + "fcm_dpo/margin": 324.9801025390625, + "fcm_dpo/q_t": 0.4211283326148987, + "grad_norm": 42.55044174194336, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.9499841928482056, + "logits/rejected": -0.9581173658370972, + "logps/chosen": -708.758544921875, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1066.9144287109375, + "loss": 1.146, + "margin_dpo/margin_mean": 324.9800720214844, + "margin_dpo/margin_std": 473.1945495605469, + "step": 599 + }, + { + "KL/chosen_KL_mean": -714.8898315429688, + "KL/mean": -876.2576904296875, + "KL/rejected_KL_mean": -1037.62548828125, + "KL/std": 520.059814453125, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0010406492510810494, + "fcm_dpo/delta": 0.06642872095108032, + "fcm_dpo/margin": 322.7356872558594, + "fcm_dpo/q_t": 0.4256163537502289, + "grad_norm": 34.19992446899414, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.0865689516067505, + "logits/rejected": -1.0850225687026978, + "logps/chosen": -781.32470703125, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1123.08203125, + "loss": 1.1766, + "margin_dpo/margin_mean": 322.73565673828125, + "margin_dpo/margin_std": 627.9359130859375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -721.1644897460938, + "KL/mean": -927.025146484375, + "KL/rejected_KL_mean": -1132.8858642578125, + "KL/std": 556.9569091796875, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0010396954603493214, + "fcm_dpo/delta": -0.029417188838124275, + "fcm_dpo/margin": 411.721435546875, + "fcm_dpo/q_t": 0.4011155664920807, + "grad_norm": 35.198631286621094, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.027404546737671, + "logits/rejected": -1.0551080703735352, + "logps/chosen": -780.298095703125, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1227.576904296875, + "loss": 1.0799, + "margin_dpo/margin_mean": 411.721435546875, + "margin_dpo/margin_std": 551.5633544921875, + "step": 601 + }, + { + "KL/chosen_KL_mean": -492.7459716796875, + "KL/mean": -736.0606689453125, + "KL/rejected_KL_mean": -979.3754272460938, + "KL/std": 500.15167236328125, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0010302650043740869, + "fcm_dpo/delta": -0.10690180212259293, + "fcm_dpo/margin": 486.62939453125, + "fcm_dpo/q_t": 0.3859218955039978, + "grad_norm": 65.08220672607422, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -1.011995553970337, + "logits/rejected": -1.0490012168884277, + "logps/chosen": -541.3394775390625, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1067.0439453125, + "loss": 1.0175, + "margin_dpo/margin_mean": 486.6294250488281, + "margin_dpo/margin_std": 531.99169921875, + "step": 602 + }, + { + "KL/chosen_KL_mean": -619.7708740234375, + "KL/mean": -846.5360107421875, + "KL/rejected_KL_mean": -1073.3011474609375, + "KL/std": 557.0963134765625, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0010079687926918268, + "fcm_dpo/delta": -0.05993969738483429, + "fcm_dpo/margin": 453.53021240234375, + "fcm_dpo/q_t": 0.39925503730773926, + "grad_norm": 37.352684020996094, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -1.0429848432540894, + "logits/rejected": -1.0421655178070068, + "logps/chosen": -690.185546875, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1173.626708984375, + "loss": 1.0769, + "margin_dpo/margin_mean": 453.53021240234375, + "margin_dpo/margin_std": 652.0963134765625, + "step": 603 + }, + { + "KL/chosen_KL_mean": -607.1182861328125, + "KL/mean": -853.111572265625, + "KL/rejected_KL_mean": -1099.1048583984375, + "KL/std": 549.8318481445312, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.000991850160062313, + "fcm_dpo/delta": -0.09249670058488846, + "fcm_dpo/margin": 491.986572265625, + "fcm_dpo/q_t": 0.3923003077507019, + "grad_norm": 32.58203125, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.9713139533996582, + "logits/rejected": -1.0068705081939697, + "logps/chosen": -653.576416015625, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1190.9593505859375, + "loss": 1.0575, + "margin_dpo/margin_mean": 491.986572265625, + "margin_dpo/margin_std": 675.6209716796875, + "step": 604 + }, + { + "KL/chosen_KL_mean": -651.98681640625, + "KL/mean": -833.0079345703125, + "KL/rejected_KL_mean": -1014.0289306640625, + "KL/std": 511.11834716796875, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.000991225242614746, + "fcm_dpo/delta": 0.04265536367893219, + "fcm_dpo/margin": 362.04205322265625, + "fcm_dpo/q_t": 0.42069005966186523, + "grad_norm": 29.918193817138672, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.9729519486427307, + "logits/rejected": -0.9916993379592896, + "logps/chosen": -718.2362060546875, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1116.333984375, + "loss": 1.1408, + "margin_dpo/margin_mean": 362.04205322265625, + "margin_dpo/margin_std": 611.448974609375, + "step": 605 + }, + { + "KL/chosen_KL_mean": -653.5445556640625, + "KL/mean": -855.438232421875, + "KL/rejected_KL_mean": -1057.3319091796875, + "KL/std": 525.418701171875, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0009987209923565388, + "fcm_dpo/delta": -0.0036588869988918304, + "fcm_dpo/margin": 403.787353515625, + "fcm_dpo/q_t": 0.40868860483169556, + "grad_norm": 30.3976993560791, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.9663287401199341, + "logits/rejected": -0.9819379448890686, + "logps/chosen": -708.3636474609375, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1155.703369140625, + "loss": 1.0962, + "margin_dpo/margin_mean": 403.7873840332031, + "margin_dpo/margin_std": 562.6357421875, + "step": 606 + }, + { + "KL/chosen_KL_mean": -667.130126953125, + "KL/mean": -832.232177734375, + "KL/rejected_KL_mean": -997.334228515625, + "KL/std": 518.4675903320312, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0010083949891850352, + "fcm_dpo/delta": 0.06891383975744247, + "fcm_dpo/margin": 330.20416259765625, + "fcm_dpo/q_t": 0.42453980445861816, + "grad_norm": 25.373777389526367, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.9681817889213562, + "logits/rejected": -0.9603500962257385, + "logps/chosen": -725.2141723632812, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1077.111328125, + "loss": 1.1554, + "margin_dpo/margin_mean": 330.20416259765625, + "margin_dpo/margin_std": 558.013671875, + "step": 607 + }, + { + "KL/chosen_KL_mean": -594.47705078125, + "KL/mean": -795.7579345703125, + "KL/rejected_KL_mean": -997.0389404296875, + "KL/std": 478.43292236328125, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0010098132770508528, + "fcm_dpo/delta": -0.006861444562673569, + "fcm_dpo/margin": 402.56182861328125, + "fcm_dpo/q_t": 0.405791699886322, + "grad_norm": 25.808034896850586, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.0151541233062744, + "logits/rejected": -1.0307934284210205, + "logps/chosen": -651.9279174804688, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1091.812255859375, + "loss": 1.0764, + "margin_dpo/margin_mean": 402.5617980957031, + "margin_dpo/margin_std": 484.75811767578125, + "step": 608 + }, + { + "KL/chosen_KL_mean": -632.0861206054688, + "KL/mean": -884.9874267578125, + "KL/rejected_KL_mean": -1137.888671875, + "KL/std": 659.6702880859375, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.0009920579614117742, + "fcm_dpo/delta": -0.10710650682449341, + "fcm_dpo/margin": 505.8026123046875, + "fcm_dpo/q_t": 0.394446462392807, + "grad_norm": 31.831369400024414, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.0399996042251587, + "logits/rejected": -1.062938928604126, + "logps/chosen": -690.8914794921875, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1226.704833984375, + "loss": 1.0632, + "margin_dpo/margin_mean": 505.8026123046875, + "margin_dpo/margin_std": 778.5198974609375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -612.4410400390625, + "KL/mean": -776.6689453125, + "KL/rejected_KL_mean": -940.8969116210938, + "KL/std": 494.581298828125, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.000998334027826786, + "fcm_dpo/delta": 0.07440324127674103, + "fcm_dpo/margin": 328.455810546875, + "fcm_dpo/q_t": 0.4244460463523865, + "grad_norm": 42.91096496582031, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.951951265335083, + "logits/rejected": -0.9316046833992004, + "logps/chosen": -678.1361083984375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1024.30224609375, + "loss": 1.1596, + "margin_dpo/margin_mean": 328.4558410644531, + "margin_dpo/margin_std": 566.2849731445312, + "step": 610 + }, + { + "KL/chosen_KL_mean": -645.9527587890625, + "KL/mean": -917.9390869140625, + "KL/rejected_KL_mean": -1189.925537109375, + "KL/std": 657.4854736328125, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0009802610147744417, + "fcm_dpo/delta": -0.14071941375732422, + "fcm_dpo/margin": 543.9727172851562, + "fcm_dpo/q_t": 0.3850560784339905, + "grad_norm": 36.08842849731445, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.0532951354980469, + "logits/rejected": -1.1042537689208984, + "logps/chosen": -698.55224609375, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1276.2564697265625, + "loss": 1.0387, + "margin_dpo/margin_mean": 543.9727783203125, + "margin_dpo/margin_std": 747.4966430664062, + "step": 611 + }, + { + "KL/chosen_KL_mean": -683.7664794921875, + "KL/mean": -897.3704833984375, + "KL/rejected_KL_mean": -1110.974609375, + "KL/std": 531.8086547851562, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0009733604965731502, + "fcm_dpo/delta": -0.016628719866275787, + "fcm_dpo/margin": 427.2080383300781, + "fcm_dpo/q_t": 0.4047042727470398, + "grad_norm": 29.613950729370117, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -0.9907621145248413, + "logits/rejected": -1.0009373426437378, + "logps/chosen": -743.0902099609375, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1199.286865234375, + "loss": 1.0815, + "margin_dpo/margin_mean": 427.2080383300781, + "margin_dpo/margin_std": 557.889892578125, + "step": 612 + }, + { + "KL/chosen_KL_mean": -618.2367553710938, + "KL/mean": -867.695068359375, + "KL/rejected_KL_mean": -1117.1533203125, + "KL/std": 589.505126953125, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0009556954028084874, + "fcm_dpo/delta": -0.08100706338882446, + "fcm_dpo/margin": 498.9165954589844, + "fcm_dpo/q_t": 0.39105772972106934, + "grad_norm": 35.980167388916016, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -0.9996154308319092, + "logits/rejected": -1.0555496215820312, + "logps/chosen": -677.9666748046875, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1222.260986328125, + "loss": 1.0421, + "margin_dpo/margin_mean": 498.9165954589844, + "margin_dpo/margin_std": 606.9149780273438, + "step": 613 + }, + { + "KL/chosen_KL_mean": -671.1931762695312, + "KL/mean": -916.2830200195312, + "KL/rejected_KL_mean": -1161.3729248046875, + "KL/std": 583.164306640625, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0009462093003094196, + "fcm_dpo/delta": -0.06686199456453323, + "fcm_dpo/margin": 490.17974853515625, + "fcm_dpo/q_t": 0.3968586027622223, + "grad_norm": 43.661155700683594, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.9862323999404907, + "logits/rejected": -1.0604286193847656, + "logps/chosen": -724.1321411132812, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1266.05224609375, + "loss": 1.0719, + "margin_dpo/margin_mean": 490.17974853515625, + "margin_dpo/margin_std": 695.0687866210938, + "step": 614 + }, + { + "KL/chosen_KL_mean": -626.55908203125, + "KL/mean": -829.8148193359375, + "KL/rejected_KL_mean": -1033.0706787109375, + "KL/std": 587.0648193359375, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0009469976648688316, + "fcm_dpo/delta": 0.015334054827690125, + "fcm_dpo/margin": 406.511474609375, + "fcm_dpo/q_t": 0.4121982157230377, + "grad_norm": 37.582454681396484, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.9607226252555847, + "logits/rejected": -0.9819501638412476, + "logps/chosen": -692.3764038085938, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1128.2481689453125, + "loss": 1.1223, + "margin_dpo/margin_mean": 406.51153564453125, + "margin_dpo/margin_std": 640.07958984375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -760.6199951171875, + "KL/mean": -923.9631958007812, + "KL/rejected_KL_mean": -1087.306396484375, + "KL/std": 499.77911376953125, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0009578055469319224, + "fcm_dpo/delta": 0.08984432369470596, + "fcm_dpo/margin": 326.68634033203125, + "fcm_dpo/q_t": 0.428780198097229, + "grad_norm": 29.95616340637207, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.0833768844604492, + "logits/rejected": -1.0702993869781494, + "logps/chosen": -825.7528686523438, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1162.0068359375, + "loss": 1.1662, + "margin_dpo/margin_mean": 326.6863708496094, + "margin_dpo/margin_std": 558.7737426757812, + "step": 616 + }, + { + "KL/chosen_KL_mean": -714.2735595703125, + "KL/mean": -834.3734130859375, + "KL/rejected_KL_mean": -954.473388671875, + "KL/std": 473.46099853515625, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0009875418618321419, + "fcm_dpo/delta": 0.16638562083244324, + "fcm_dpo/margin": 240.19979858398438, + "fcm_dpo/q_t": 0.44630277156829834, + "grad_norm": 62.97007369995117, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.9708187580108643, + "logits/rejected": -0.9448531866073608, + "logps/chosen": -777.2791137695312, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1018.7074584960938, + "loss": 1.234, + "margin_dpo/margin_mean": 240.19979858398438, + "margin_dpo/margin_std": 552.0235595703125, + "step": 617 + }, + { + "KL/chosen_KL_mean": -734.6864013671875, + "KL/mean": -995.43408203125, + "KL/rejected_KL_mean": -1256.181884765625, + "KL/std": 642.0171508789062, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.000983049045316875, + "fcm_dpo/delta": -0.11882639676332474, + "fcm_dpo/margin": 521.495361328125, + "fcm_dpo/q_t": 0.39003878831863403, + "grad_norm": 40.660770416259766, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -0.9848862886428833, + "logits/rejected": -1.026947021484375, + "logps/chosen": -801.7877807617188, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1348.335205078125, + "loss": 1.0813, + "margin_dpo/margin_mean": 521.495361328125, + "margin_dpo/margin_std": 820.2125244140625, + "step": 618 + }, + { + "KL/chosen_KL_mean": -742.6043701171875, + "KL/mean": -938.7860107421875, + "KL/rejected_KL_mean": -1134.9677734375, + "KL/std": 593.3043212890625, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0009737603249959648, + "fcm_dpo/delta": 0.018636221066117287, + "fcm_dpo/margin": 392.3634033203125, + "fcm_dpo/q_t": 0.42084750533103943, + "grad_norm": 54.80867004394531, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -0.9916151762008667, + "logits/rejected": -1.0047380924224854, + "logps/chosen": -798.5825805664062, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1228.153076171875, + "loss": 1.1708, + "margin_dpo/margin_mean": 392.3633728027344, + "margin_dpo/margin_std": 789.0343627929688, + "step": 619 + }, + { + "KL/chosen_KL_mean": -672.1712646484375, + "KL/mean": -863.21923828125, + "KL/rejected_KL_mean": -1054.267333984375, + "KL/std": 537.2314453125, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0009749538148753345, + "fcm_dpo/delta": 0.028370080515742302, + "fcm_dpo/margin": 382.0960693359375, + "fcm_dpo/q_t": 0.4163762927055359, + "grad_norm": 40.9302978515625, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.0421342849731445, + "logits/rejected": -1.0519472360610962, + "logps/chosen": -731.96875, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1132.677978515625, + "loss": 1.1308, + "margin_dpo/margin_mean": 382.0960388183594, + "margin_dpo/margin_std": 612.406005859375, + "step": 620 + }, + { + "KL/chosen_KL_mean": -655.530029296875, + "KL/mean": -934.019287109375, + "KL/rejected_KL_mean": -1212.508544921875, + "KL/std": 637.230712890625, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0009663389064371586, + "fcm_dpo/delta": -0.14617975056171417, + "fcm_dpo/margin": 556.978515625, + "fcm_dpo/q_t": 0.37928950786590576, + "grad_norm": 52.21852111816406, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.0842504501342773, + "logits/rejected": -1.1371203660964966, + "logps/chosen": -709.4637451171875, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1300.8779296875, + "loss": 1.027, + "margin_dpo/margin_mean": 556.978515625, + "margin_dpo/margin_std": 705.281005859375, + "step": 621 + }, + { + "KL/chosen_KL_mean": -647.168212890625, + "KL/mean": -839.0968017578125, + "KL/rejected_KL_mean": -1031.025390625, + "KL/std": 493.4986267089844, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.000951895082835108, + "fcm_dpo/delta": 0.03557223081588745, + "fcm_dpo/margin": 383.8570861816406, + "fcm_dpo/q_t": 0.418539822101593, + "grad_norm": 34.71763610839844, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.9290555715560913, + "logits/rejected": -0.9212709665298462, + "logps/chosen": -707.4541015625, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1116.5440673828125, + "loss": 1.1348, + "margin_dpo/margin_mean": 383.85711669921875, + "margin_dpo/margin_std": 628.03955078125, + "step": 622 + }, + { + "KL/chosen_KL_mean": -708.8759765625, + "KL/mean": -862.5560302734375, + "KL/rejected_KL_mean": -1016.2362060546875, + "KL/std": 500.7294616699219, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0009737034561112523, + "fcm_dpo/delta": 0.10380120575428009, + "fcm_dpo/margin": 307.3602294921875, + "fcm_dpo/q_t": 0.433912456035614, + "grad_norm": 36.33531951904297, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.0603008270263672, + "logits/rejected": -1.0622398853302002, + "logps/chosen": -773.0328979492188, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1101.3192138671875, + "loss": 1.1903, + "margin_dpo/margin_mean": 307.3602294921875, + "margin_dpo/margin_std": 612.846435546875, + "step": 623 + }, + { + "KL/chosen_KL_mean": -698.8944091796875, + "KL/mean": -934.23046875, + "KL/rejected_KL_mean": -1169.5665283203125, + "KL/std": 536.6535034179688, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0009705376578494906, + "fcm_dpo/delta": -0.05949598550796509, + "fcm_dpo/margin": 470.67218017578125, + "fcm_dpo/q_t": 0.3935359716415405, + "grad_norm": 32.372291564941406, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.0685322284698486, + "logits/rejected": -1.080263376235962, + "logps/chosen": -770.81298828125, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1266.6986083984375, + "loss": 1.0555, + "margin_dpo/margin_mean": 470.67218017578125, + "margin_dpo/margin_std": 580.5545043945312, + "step": 624 + }, + { + "KL/chosen_KL_mean": -654.531982421875, + "KL/mean": -915.96875, + "KL/rejected_KL_mean": -1177.405517578125, + "KL/std": 583.6700439453125, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0009549872484058142, + "fcm_dpo/delta": -0.10442975163459778, + "fcm_dpo/margin": 522.87353515625, + "fcm_dpo/q_t": 0.3861393332481384, + "grad_norm": 65.17060089111328, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.9756132960319519, + "logits/rejected": -1.005097508430481, + "logps/chosen": -712.8740234375, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1263.495849609375, + "loss": 1.0116, + "margin_dpo/margin_mean": 522.87353515625, + "margin_dpo/margin_std": 558.5311889648438, + "step": 625 + }, + { + "KL/chosen_KL_mean": -834.64990234375, + "KL/mean": -981.01416015625, + "KL/rejected_KL_mean": -1127.37841796875, + "KL/std": 656.3546752929688, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.00096412020502612, + "fcm_dpo/delta": 0.12122038006782532, + "fcm_dpo/margin": 292.728515625, + "fcm_dpo/q_t": 0.43377774953842163, + "grad_norm": 33.22077178955078, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.9854834079742432, + "logits/rejected": -0.9787349700927734, + "logps/chosen": -909.762451171875, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1226.567138671875, + "loss": 1.219, + "margin_dpo/margin_mean": 292.728515625, + "margin_dpo/margin_std": 642.291015625, + "step": 626 + }, + { + "KL/chosen_KL_mean": -557.949951171875, + "KL/mean": -840.2698974609375, + "KL/rejected_KL_mean": -1122.58984375, + "KL/std": 673.544189453125, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0009557720040902495, + "fcm_dpo/delta": -0.14757977426052094, + "fcm_dpo/margin": 564.6399536132812, + "fcm_dpo/q_t": 0.3869680166244507, + "grad_norm": 30.814462661743164, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.9499855041503906, + "logits/rejected": -1.0196809768676758, + "logps/chosen": -605.693115234375, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1229.3443603515625, + "loss": 1.0355, + "margin_dpo/margin_mean": 564.6399536132812, + "margin_dpo/margin_std": 768.2205810546875, + "step": 627 + }, + { + "KL/chosen_KL_mean": -751.0343017578125, + "KL/mean": -974.2967529296875, + "KL/rejected_KL_mean": -1197.55908203125, + "KL/std": 584.6043090820312, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0009376833913847804, + "fcm_dpo/delta": -0.019609639421105385, + "fcm_dpo/margin": 446.5248718261719, + "fcm_dpo/q_t": 0.4066890776157379, + "grad_norm": 30.563941955566406, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.070842981338501, + "logits/rejected": -1.0892653465270996, + "logps/chosen": -811.2172241210938, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1299.11376953125, + "loss": 1.1021, + "margin_dpo/margin_mean": 446.5248718261719, + "margin_dpo/margin_std": 670.14599609375, + "step": 628 + }, + { + "KL/chosen_KL_mean": -705.7962646484375, + "KL/mean": -912.211669921875, + "KL/rejected_KL_mean": -1118.626953125, + "KL/std": 545.0255126953125, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0009402513969689608, + "fcm_dpo/delta": 0.01231019850820303, + "fcm_dpo/margin": 412.83074951171875, + "fcm_dpo/q_t": 0.40944704413414, + "grad_norm": 39.234859466552734, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.059785008430481, + "logits/rejected": -1.0810933113098145, + "logps/chosen": -770.009765625, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1210.2806396484375, + "loss": 1.0966, + "margin_dpo/margin_mean": 412.83074951171875, + "margin_dpo/margin_std": 540.1163330078125, + "step": 629 + }, + { + "KL/chosen_KL_mean": -643.7701416015625, + "KL/mean": -760.624755859375, + "KL/rejected_KL_mean": -877.4793701171875, + "KL/std": 562.6679077148438, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0009703253163024783, + "fcm_dpo/delta": 0.17595510184764862, + "fcm_dpo/margin": 233.70913696289062, + "fcm_dpo/q_t": 0.45422884821891785, + "grad_norm": 62.001991271972656, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -0.9804481863975525, + "logits/rejected": -0.9580028653144836, + "logps/chosen": -703.0611572265625, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -961.07763671875, + "loss": 1.2694, + "margin_dpo/margin_mean": 233.70913696289062, + "margin_dpo/margin_std": 673.7122802734375, + "step": 630 + }, + { + "KL/chosen_KL_mean": -785.5379028320312, + "KL/mean": -969.3244018554688, + "KL/rejected_KL_mean": -1153.11083984375, + "KL/std": 567.6865844726562, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0009818272665143013, + "fcm_dpo/delta": 0.04045334458351135, + "fcm_dpo/margin": 367.57293701171875, + "fcm_dpo/q_t": 0.4174206256866455, + "grad_norm": 37.814369201660156, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.0803989171981812, + "logits/rejected": -1.0849816799163818, + "logps/chosen": -844.9915161132812, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1234.0623779296875, + "loss": 1.1542, + "margin_dpo/margin_mean": 367.57293701171875, + "margin_dpo/margin_std": 651.4695434570312, + "step": 631 + }, + { + "KL/chosen_KL_mean": -670.6087646484375, + "KL/mean": -863.7804565429688, + "KL/rejected_KL_mean": -1056.9521484375, + "KL/std": 512.10205078125, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.0009807306341826916, + "fcm_dpo/delta": 0.021565694361925125, + "fcm_dpo/margin": 386.3433532714844, + "fcm_dpo/q_t": 0.4137539267539978, + "grad_norm": 38.96223449707031, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.0882303714752197, + "logits/rejected": -1.0942792892456055, + "logps/chosen": -731.9603271484375, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1143.1123046875, + "loss": 1.131, + "margin_dpo/margin_mean": 386.3433532714844, + "margin_dpo/margin_std": 618.545166015625, + "step": 632 + }, + { + "KL/chosen_KL_mean": -748.426513671875, + "KL/mean": -891.647705078125, + "KL/rejected_KL_mean": -1034.868896484375, + "KL/std": 514.609619140625, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0010064642410725355, + "fcm_dpo/delta": 0.11451433598995209, + "fcm_dpo/margin": 286.4423828125, + "fcm_dpo/q_t": 0.43422216176986694, + "grad_norm": 49.707305908203125, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.1021374464035034, + "logits/rejected": -1.1128813028335571, + "logps/chosen": -805.7047119140625, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1126.452880859375, + "loss": 1.2029, + "margin_dpo/margin_mean": 286.4424133300781, + "margin_dpo/margin_std": 591.9028930664062, + "step": 633 + }, + { + "KL/chosen_KL_mean": -853.5125732421875, + "KL/mean": -984.5179443359375, + "KL/rejected_KL_mean": -1115.5233154296875, + "KL/std": 621.7991943359375, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0010286369360983372, + "fcm_dpo/delta": 0.13422925770282745, + "fcm_dpo/margin": 262.0107727050781, + "fcm_dpo/q_t": 0.4410145878791809, + "grad_norm": 37.784141540527344, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.9951022863388062, + "logits/rejected": -0.9891289472579956, + "logps/chosen": -920.1315307617188, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1222.64892578125, + "loss": 1.2209, + "margin_dpo/margin_mean": 262.01080322265625, + "margin_dpo/margin_std": 593.56982421875, + "step": 634 + }, + { + "KL/chosen_KL_mean": -727.9835205078125, + "KL/mean": -902.00439453125, + "KL/rejected_KL_mean": -1076.025146484375, + "KL/std": 627.1515502929688, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.001042095711454749, + "fcm_dpo/delta": 0.038728609681129456, + "fcm_dpo/margin": 348.04168701171875, + "fcm_dpo/q_t": 0.42095375061035156, + "grad_norm": 47.4949951171875, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.052489995956421, + "logits/rejected": -1.0338871479034424, + "logps/chosen": -801.9390258789062, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1158.525634765625, + "loss": 1.1538, + "margin_dpo/margin_mean": 348.04168701171875, + "margin_dpo/margin_std": 638.2135009765625, + "step": 635 + }, + { + "KL/chosen_KL_mean": -695.1549072265625, + "KL/mean": -903.6052856445312, + "KL/rejected_KL_mean": -1112.0556640625, + "KL/std": 621.8355712890625, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0010470494162291288, + "fcm_dpo/delta": -0.039131179451942444, + "fcm_dpo/margin": 416.9007263183594, + "fcm_dpo/q_t": 0.40514853596687317, + "grad_norm": 30.509817123413086, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.9890474081039429, + "logits/rejected": -0.9911370277404785, + "logps/chosen": -754.7838134765625, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1194.034423828125, + "loss": 1.092, + "margin_dpo/margin_mean": 416.90069580078125, + "margin_dpo/margin_std": 636.7688598632812, + "step": 636 + }, + { + "KL/chosen_KL_mean": -683.0616455078125, + "KL/mean": -917.3276977539062, + "KL/rejected_KL_mean": -1151.59375, + "KL/std": 613.7899169921875, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.001019324641674757, + "fcm_dpo/delta": -0.08231889456510544, + "fcm_dpo/margin": 468.5321044921875, + "fcm_dpo/q_t": 0.39453592896461487, + "grad_norm": 30.800790786743164, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.0669300556182861, + "logits/rejected": -1.1149356365203857, + "logps/chosen": -732.71435546875, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1249.998779296875, + "loss": 1.0572, + "margin_dpo/margin_mean": 468.5321044921875, + "margin_dpo/margin_std": 635.0623779296875, + "step": 637 + }, + { + "KL/chosen_KL_mean": -668.5267333984375, + "KL/mean": -855.3203125, + "KL/rejected_KL_mean": -1042.1138916015625, + "KL/std": 581.373779296875, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0010332321980968118, + "fcm_dpo/delta": 0.011849157512187958, + "fcm_dpo/margin": 373.587158203125, + "fcm_dpo/q_t": 0.407895565032959, + "grad_norm": 35.188201904296875, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.9733033180236816, + "logits/rejected": -0.9613117575645447, + "logps/chosen": -726.683349609375, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1121.415283203125, + "loss": 1.1525, + "margin_dpo/margin_mean": 373.587158203125, + "margin_dpo/margin_std": 662.7489013671875, + "step": 638 + }, + { + "KL/chosen_KL_mean": -904.2991943359375, + "KL/mean": -982.6353759765625, + "KL/rejected_KL_mean": -1060.9715576171875, + "KL/std": 535.6351318359375, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.001059873029589653, + "fcm_dpo/delta": 0.2377687692642212, + "fcm_dpo/margin": 156.67237854003906, + "fcm_dpo/q_t": 0.46347498893737793, + "grad_norm": 126.2874526977539, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.9430060982704163, + "logits/rejected": -0.8850451707839966, + "logps/chosen": -976.622314453125, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1135.2464599609375, + "loss": 1.3494, + "margin_dpo/margin_mean": 156.67239379882812, + "margin_dpo/margin_std": 678.263427734375, + "step": 639 + }, + { + "KL/chosen_KL_mean": -701.374755859375, + "KL/mean": -978.5585327148438, + "KL/rejected_KL_mean": -1255.742431640625, + "KL/std": 643.263427734375, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0010469657136127353, + "fcm_dpo/delta": -0.19107630848884583, + "fcm_dpo/margin": 554.3675537109375, + "fcm_dpo/q_t": 0.3767038881778717, + "grad_norm": 44.614402770996094, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.9669941067695618, + "logits/rejected": -1.0004709959030151, + "logps/chosen": -757.5091552734375, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1364.342529296875, + "loss": 1.0195, + "margin_dpo/margin_mean": 554.3675537109375, + "margin_dpo/margin_std": 752.642578125, + "step": 640 + }, + { + "KL/chosen_KL_mean": -816.3616943359375, + "KL/mean": -995.03759765625, + "KL/rejected_KL_mean": -1173.71337890625, + "KL/std": 553.8012084960938, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0010342487366870046, + "fcm_dpo/delta": 0.031536415219306946, + "fcm_dpo/margin": 357.3519287109375, + "fcm_dpo/q_t": 0.4202731251716614, + "grad_norm": 37.84084701538086, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.0097227096557617, + "logits/rejected": -1.0219624042510986, + "logps/chosen": -881.3585205078125, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1260.705810546875, + "loss": 1.175, + "margin_dpo/margin_mean": 357.3519287109375, + "margin_dpo/margin_std": 727.092041015625, + "step": 641 + }, + { + "KL/chosen_KL_mean": -783.3552856445312, + "KL/mean": -1002.3966674804688, + "KL/rejected_KL_mean": -1221.43798828125, + "KL/std": 728.3822021484375, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.001035279594361782, + "fcm_dpo/delta": -0.05637134611606598, + "fcm_dpo/margin": 438.0827941894531, + "fcm_dpo/q_t": 0.4100719690322876, + "grad_norm": 49.01300811767578, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.9654021263122559, + "logits/rejected": -1.00310480594635, + "logps/chosen": -849.0445556640625, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1331.68017578125, + "loss": 1.1483, + "margin_dpo/margin_mean": 438.0827941894531, + "margin_dpo/margin_std": 883.371826171875, + "step": 642 + }, + { + "KL/chosen_KL_mean": -696.7486572265625, + "KL/mean": -870.5406494140625, + "KL/rejected_KL_mean": -1044.33251953125, + "KL/std": 529.383056640625, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.001025365199893713, + "fcm_dpo/delta": 0.04426843300461769, + "fcm_dpo/margin": 347.5838623046875, + "fcm_dpo/q_t": 0.4190395772457123, + "grad_norm": 49.06093215942383, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.1120374202728271, + "logits/rejected": -1.1139336824417114, + "logps/chosen": -748.6986694335938, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1131.8009033203125, + "loss": 1.1386, + "margin_dpo/margin_mean": 347.5838623046875, + "margin_dpo/margin_std": 536.2342529296875, + "step": 643 + }, + { + "KL/chosen_KL_mean": -683.4932250976562, + "KL/mean": -854.1671142578125, + "KL/rejected_KL_mean": -1024.841064453125, + "KL/std": 624.2161865234375, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0010422810446470976, + "fcm_dpo/delta": 0.045872341841459274, + "fcm_dpo/margin": 341.3477478027344, + "fcm_dpo/q_t": 0.42317748069763184, + "grad_norm": 61.52754211425781, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -0.9743162393569946, + "logits/rejected": -0.9815078973770142, + "logps/chosen": -742.5108642578125, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1111.977783203125, + "loss": 1.1899, + "margin_dpo/margin_mean": 341.3477478027344, + "margin_dpo/margin_std": 716.8045654296875, + "step": 644 + }, + { + "KL/chosen_KL_mean": -720.267333984375, + "KL/mean": -811.9573974609375, + "KL/rejected_KL_mean": -903.6475830078125, + "KL/std": 495.16522216796875, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.001072922721505165, + "fcm_dpo/delta": 0.20786070823669434, + "fcm_dpo/margin": 183.38027954101562, + "fcm_dpo/q_t": 0.45883452892303467, + "grad_norm": 105.2673110961914, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.9859621524810791, + "logits/rejected": -0.9741485118865967, + "logps/chosen": -776.143310546875, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1001.4284057617188, + "loss": 1.327, + "margin_dpo/margin_mean": 183.3802490234375, + "margin_dpo/margin_std": 663.9035034179688, + "step": 645 + }, + { + "KL/chosen_KL_mean": -685.960693359375, + "KL/mean": -840.7113037109375, + "KL/rejected_KL_mean": -995.4619140625, + "KL/std": 474.540283203125, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0010956395417451859, + "fcm_dpo/delta": 0.06295713782310486, + "fcm_dpo/margin": 309.5010986328125, + "fcm_dpo/q_t": 0.4245607852935791, + "grad_norm": 59.16865539550781, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.0069502592086792, + "logits/rejected": -0.9910367727279663, + "logps/chosen": -747.2364501953125, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1072.9676513671875, + "loss": 1.1686, + "margin_dpo/margin_mean": 309.5011291503906, + "margin_dpo/margin_std": 567.9707641601562, + "step": 646 + }, + { + "KL/chosen_KL_mean": -598.7654418945312, + "KL/mean": -790.0628662109375, + "KL/rejected_KL_mean": -981.3603515625, + "KL/std": 536.9503784179688, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0010998575016856194, + "fcm_dpo/delta": -0.021717313677072525, + "fcm_dpo/margin": 382.5949401855469, + "fcm_dpo/q_t": 0.40425243973731995, + "grad_norm": 30.720497131347656, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.9346556067466736, + "logits/rejected": -0.966040313243866, + "logps/chosen": -653.6178588867188, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1074.8797607421875, + "loss": 1.09, + "margin_dpo/margin_mean": 382.5949401855469, + "margin_dpo/margin_std": 533.8380126953125, + "step": 647 + }, + { + "KL/chosen_KL_mean": -623.2069091796875, + "KL/mean": -905.6817626953125, + "KL/rejected_KL_mean": -1188.156494140625, + "KL/std": 638.7603759765625, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0010651691118255258, + "fcm_dpo/delta": -0.21454653143882751, + "fcm_dpo/margin": 564.94970703125, + "fcm_dpo/q_t": 0.3732055723667145, + "grad_norm": 32.451690673828125, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.0365365743637085, + "logits/rejected": -1.094492793083191, + "logps/chosen": -677.37841796875, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1286.869384765625, + "loss": 1.0033, + "margin_dpo/margin_mean": 564.94970703125, + "margin_dpo/margin_std": 759.1719360351562, + "step": 648 + }, + { + "KL/chosen_KL_mean": -653.470703125, + "KL/mean": -790.299072265625, + "KL/rejected_KL_mean": -927.12744140625, + "KL/std": 504.5975341796875, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.001073287334293127, + "fcm_dpo/delta": 0.10895916074514389, + "fcm_dpo/margin": 273.65667724609375, + "fcm_dpo/q_t": 0.43835365772247314, + "grad_norm": 32.83501434326172, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.073495626449585, + "logits/rejected": -1.067068338394165, + "logps/chosen": -715.9510498046875, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1007.20458984375, + "loss": 1.2198, + "margin_dpo/margin_mean": 273.6567077636719, + "margin_dpo/margin_std": 615.2674560546875, + "step": 649 + }, + { + "KL/chosen_KL_mean": -694.3304443359375, + "KL/mean": -907.985107421875, + "KL/rejected_KL_mean": -1121.6396484375, + "KL/std": 614.8465576171875, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0010633300989866257, + "fcm_dpo/delta": -0.057344742119312286, + "fcm_dpo/margin": 427.30926513671875, + "fcm_dpo/q_t": 0.4013846814632416, + "grad_norm": 37.21390914916992, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.0348689556121826, + "logits/rejected": -1.058072805404663, + "logps/chosen": -750.4232177734375, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1219.904541015625, + "loss": 1.1061, + "margin_dpo/margin_mean": 427.3092346191406, + "margin_dpo/margin_std": 701.234375, + "step": 650 + }, + { + "KL/chosen_KL_mean": -467.93243408203125, + "KL/mean": -738.3851318359375, + "KL/rejected_KL_mean": -1008.8378295898438, + "KL/std": 570.8089599609375, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0010359440930187702, + "fcm_dpo/delta": -0.17007434368133545, + "fcm_dpo/margin": 540.9053955078125, + "fcm_dpo/q_t": 0.3740183115005493, + "grad_norm": 34.63212966918945, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.0335466861724854, + "logits/rejected": -1.0901474952697754, + "logps/chosen": -511.35784912109375, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1108.795654296875, + "loss": 0.9833, + "margin_dpo/margin_mean": 540.9054565429688, + "margin_dpo/margin_std": 587.92236328125, + "step": 651 + }, + { + "KL/chosen_KL_mean": -602.069580078125, + "KL/mean": -803.7984619140625, + "KL/rejected_KL_mean": -1005.5272827148438, + "KL/std": 584.66357421875, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0010270553175359964, + "fcm_dpo/delta": -0.01501021534204483, + "fcm_dpo/margin": 403.45770263671875, + "fcm_dpo/q_t": 0.40905457735061646, + "grad_norm": 35.15320587158203, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.0242347717285156, + "logits/rejected": -1.0521032810211182, + "logps/chosen": -664.6463623046875, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1117.295166015625, + "loss": 1.1198, + "margin_dpo/margin_mean": 403.4577331542969, + "margin_dpo/margin_std": 674.022705078125, + "step": 652 + }, + { + "KL/chosen_KL_mean": -741.655029296875, + "KL/mean": -929.6370239257812, + "KL/rejected_KL_mean": -1117.618896484375, + "KL/std": 622.3851318359375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0010281873401254416, + "fcm_dpo/delta": 0.013938883319497108, + "fcm_dpo/margin": 375.96380615234375, + "fcm_dpo/q_t": 0.41611939668655396, + "grad_norm": 37.6129150390625, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.0994454622268677, + "logits/rejected": -1.1240202188491821, + "logps/chosen": -802.76806640625, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1220.8685302734375, + "loss": 1.1391, + "margin_dpo/margin_mean": 375.96380615234375, + "margin_dpo/margin_std": 655.464599609375, + "step": 653 + }, + { + "KL/chosen_KL_mean": -672.4105224609375, + "KL/mean": -853.2705078125, + "KL/rejected_KL_mean": -1034.13037109375, + "KL/std": 502.23895263671875, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0010315109975636005, + "fcm_dpo/delta": 0.027932938188314438, + "fcm_dpo/margin": 361.7199401855469, + "fcm_dpo/q_t": 0.41764745116233826, + "grad_norm": 39.66285705566406, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.0680111646652222, + "logits/rejected": -1.083982229232788, + "logps/chosen": -734.138671875, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1132.904296875, + "loss": 1.1303, + "margin_dpo/margin_mean": 361.71990966796875, + "margin_dpo/margin_std": 583.1453857421875, + "step": 654 + }, + { + "KL/chosen_KL_mean": -620.1411743164062, + "KL/mean": -814.458251953125, + "KL/rejected_KL_mean": -1008.7752685546875, + "KL/std": 511.49462890625, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0010331221856176853, + "fcm_dpo/delta": -0.001570335472933948, + "fcm_dpo/margin": 388.6340637207031, + "fcm_dpo/q_t": 0.4107121527194977, + "grad_norm": 32.915672302246094, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.0347508192062378, + "logits/rejected": -1.07468581199646, + "logps/chosen": -669.718017578125, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1107.067138671875, + "loss": 1.1139, + "margin_dpo/margin_mean": 388.6340637207031, + "margin_dpo/margin_std": 605.7490234375, + "step": 655 + }, + { + "KL/chosen_KL_mean": -692.3442993164062, + "KL/mean": -1017.2608642578125, + "KL/rejected_KL_mean": -1342.177490234375, + "KL/std": 688.7314453125, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.000997279305011034, + "fcm_dpo/delta": -0.26455968618392944, + "fcm_dpo/margin": 649.8331298828125, + "fcm_dpo/q_t": 0.3588838279247284, + "grad_norm": 84.29036712646484, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.9399580359458923, + "logits/rejected": -1.0097756385803223, + "logps/chosen": -744.8937377929688, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1455.85205078125, + "loss": 0.9431, + "margin_dpo/margin_mean": 649.8331298828125, + "margin_dpo/margin_std": 701.1928100585938, + "step": 656 + }, + { + "KL/chosen_KL_mean": -637.6719970703125, + "KL/mean": -896.0465698242188, + "KL/rejected_KL_mean": -1154.421142578125, + "KL/std": 654.101806640625, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0009633610025048256, + "fcm_dpo/delta": -0.10318416357040405, + "fcm_dpo/margin": 516.7491455078125, + "fcm_dpo/q_t": 0.3883362412452698, + "grad_norm": 40.34549331665039, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.0887951850891113, + "logits/rejected": -1.135094404220581, + "logps/chosen": -684.37255859375, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1252.3359375, + "loss": 1.0432, + "margin_dpo/margin_mean": 516.7491455078125, + "margin_dpo/margin_std": 678.4373779296875, + "step": 657 + }, + { + "KL/chosen_KL_mean": -683.7800903320312, + "KL/mean": -892.7841796875, + "KL/rejected_KL_mean": -1101.788330078125, + "KL/std": 510.96685791015625, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0009549233363941312, + "fcm_dpo/delta": 0.0002949535846710205, + "fcm_dpo/margin": 418.00811767578125, + "fcm_dpo/q_t": 0.4085996150970459, + "grad_norm": 33.80031967163086, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -1.0111554861068726, + "logits/rejected": -1.040154218673706, + "logps/chosen": -744.73828125, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1197.727783203125, + "loss": 1.1024, + "margin_dpo/margin_mean": 418.0080871582031, + "margin_dpo/margin_std": 586.8758544921875, + "step": 658 + }, + { + "KL/chosen_KL_mean": -599.9077758789062, + "KL/mean": -799.1854858398438, + "KL/rejected_KL_mean": -998.4631958007812, + "KL/std": 521.9739990234375, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0009595429291948676, + "fcm_dpo/delta": 0.01811748929321766, + "fcm_dpo/margin": 398.55535888671875, + "fcm_dpo/q_t": 0.41348063945770264, + "grad_norm": 36.021488189697266, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.9547412395477295, + "logits/rejected": -0.9358307123184204, + "logps/chosen": -676.6507568359375, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1085.93408203125, + "loss": 1.1078, + "margin_dpo/margin_mean": 398.5553894042969, + "margin_dpo/margin_std": 565.2937622070312, + "step": 659 + }, + { + "KL/chosen_KL_mean": -662.3392333984375, + "KL/mean": -914.98876953125, + "KL/rejected_KL_mean": -1167.63818359375, + "KL/std": 599.0900268554688, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0009526328067295253, + "fcm_dpo/delta": -0.08540107309818268, + "fcm_dpo/margin": 505.2990417480469, + "fcm_dpo/q_t": 0.39148586988449097, + "grad_norm": 50.54105758666992, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.0727956295013428, + "logits/rejected": -1.0982887744903564, + "logps/chosen": -721.3871459960938, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1243.598388671875, + "loss": 1.0471, + "margin_dpo/margin_mean": 505.2990417480469, + "margin_dpo/margin_std": 646.0845947265625, + "step": 660 + }, + { + "KL/chosen_KL_mean": -582.4307861328125, + "KL/mean": -812.2855224609375, + "KL/rejected_KL_mean": -1042.1402587890625, + "KL/std": 655.1251220703125, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0009459180873818696, + "fcm_dpo/delta": -0.03662613034248352, + "fcm_dpo/margin": 459.70941162109375, + "fcm_dpo/q_t": 0.4045267105102539, + "grad_norm": 42.704566955566406, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.0439341068267822, + "logits/rejected": -1.060086727142334, + "logps/chosen": -633.104736328125, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1128.14599609375, + "loss": 1.0808, + "margin_dpo/margin_mean": 459.70941162109375, + "margin_dpo/margin_std": 665.7325439453125, + "step": 661 + }, + { + "KL/chosen_KL_mean": -672.673583984375, + "KL/mean": -856.9928588867188, + "KL/rejected_KL_mean": -1041.312255859375, + "KL/std": 552.8853149414062, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0009478401625528932, + "fcm_dpo/delta": 0.0524093359708786, + "fcm_dpo/margin": 368.6386413574219, + "fcm_dpo/q_t": 0.4219122529029846, + "grad_norm": 29.389673233032227, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.998336136341095, + "logits/rejected": -1.0048918724060059, + "logps/chosen": -741.9346313476562, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1130.3681640625, + "loss": 1.1593, + "margin_dpo/margin_mean": 368.63861083984375, + "margin_dpo/margin_std": 669.459228515625, + "step": 662 + }, + { + "KL/chosen_KL_mean": -634.3244018554688, + "KL/mean": -851.9996337890625, + "KL/rejected_KL_mean": -1069.675048828125, + "KL/std": 624.066162109375, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0009467662894167006, + "fcm_dpo/delta": -0.01277830172330141, + "fcm_dpo/margin": 435.3505554199219, + "fcm_dpo/q_t": 0.4100111126899719, + "grad_norm": 28.35548210144043, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.0652190446853638, + "logits/rejected": -1.0904886722564697, + "logps/chosen": -699.2033081054688, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1183.600341796875, + "loss": 1.1182, + "margin_dpo/margin_mean": 435.3505554199219, + "margin_dpo/margin_std": 725.4118041992188, + "step": 663 + }, + { + "KL/chosen_KL_mean": -676.7174072265625, + "KL/mean": -928.8491821289062, + "KL/rejected_KL_mean": -1180.98095703125, + "KL/std": 632.78857421875, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0009336504153907299, + "fcm_dpo/delta": -0.07489388436079025, + "fcm_dpo/margin": 504.2635192871094, + "fcm_dpo/q_t": 0.39537733793258667, + "grad_norm": 28.93507194519043, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.0151818990707397, + "logits/rejected": -1.051823616027832, + "logps/chosen": -737.6058959960938, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1286.502685546875, + "loss": 1.0592, + "margin_dpo/margin_mean": 504.2635192871094, + "margin_dpo/margin_std": 676.779541015625, + "step": 664 + }, + { + "KL/chosen_KL_mean": -594.496826171875, + "KL/mean": -803.6724853515625, + "KL/rejected_KL_mean": -1012.8480834960938, + "KL/std": 522.0977783203125, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.000934773008339107, + "fcm_dpo/delta": 0.009301244281232357, + "fcm_dpo/margin": 418.3512878417969, + "fcm_dpo/q_t": 0.4094030261039734, + "grad_norm": 54.36912536621094, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.0966243743896484, + "logits/rejected": -1.1229465007781982, + "logps/chosen": -655.0609130859375, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1097.656982421875, + "loss": 1.0875, + "margin_dpo/margin_mean": 418.351318359375, + "margin_dpo/margin_std": 510.9215087890625, + "step": 665 + }, + { + "KL/chosen_KL_mean": -634.3648681640625, + "KL/mean": -842.7044677734375, + "KL/rejected_KL_mean": -1051.0439453125, + "KL/std": 519.7288818359375, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0009357793023809791, + "fcm_dpo/delta": 0.010474386624991894, + "fcm_dpo/margin": 416.6790771484375, + "fcm_dpo/q_t": 0.4114568829536438, + "grad_norm": 32.34466552734375, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.908434271812439, + "logits/rejected": -0.9176754951477051, + "logps/chosen": -698.7847900390625, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1146.935546875, + "loss": 1.1038, + "margin_dpo/margin_mean": 416.6790771484375, + "margin_dpo/margin_std": 576.046142578125, + "step": 666 + }, + { + "KL/chosen_KL_mean": -723.91455078125, + "KL/mean": -971.6058349609375, + "KL/rejected_KL_mean": -1219.2972412109375, + "KL/std": 578.0843505859375, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0009265096159651875, + "fcm_dpo/delta": -0.062080685049295425, + "fcm_dpo/margin": 495.38262939453125, + "fcm_dpo/q_t": 0.39601364731788635, + "grad_norm": 40.060569763183594, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.9687680006027222, + "logits/rejected": -0.9711791276931763, + "logps/chosen": -793.1915283203125, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1307.1326904296875, + "loss": 1.0586, + "margin_dpo/margin_mean": 495.38262939453125, + "margin_dpo/margin_std": 638.7156982421875, + "step": 667 + }, + { + "KL/chosen_KL_mean": -778.2903442382812, + "KL/mean": -945.34033203125, + "KL/rejected_KL_mean": -1112.3902587890625, + "KL/std": 616.9835205078125, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0009423915180377662, + "fcm_dpo/delta": 0.08703204989433289, + "fcm_dpo/margin": 334.09991455078125, + "fcm_dpo/q_t": 0.43206775188446045, + "grad_norm": 48.35491943359375, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.0378694534301758, + "logits/rejected": -1.0449373722076416, + "logps/chosen": -850.8943481445312, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1216.12939453125, + "loss": 1.2061, + "margin_dpo/margin_mean": 334.09991455078125, + "margin_dpo/margin_std": 731.4624633789062, + "step": 668 + }, + { + "KL/chosen_KL_mean": -578.2383422851562, + "KL/mean": -827.3543090820312, + "KL/rejected_KL_mean": -1076.47021484375, + "KL/std": 574.401611328125, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0009338557720184326, + "fcm_dpo/delta": -0.06840614229440689, + "fcm_dpo/margin": 498.23193359375, + "fcm_dpo/q_t": 0.39426541328430176, + "grad_norm": 26.005155563354492, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.00222909450531, + "logits/rejected": -1.0140796899795532, + "logps/chosen": -624.354736328125, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1154.39453125, + "loss": 1.0568, + "margin_dpo/margin_mean": 498.23193359375, + "margin_dpo/margin_std": 644.14208984375, + "step": 669 + }, + { + "KL/chosen_KL_mean": -571.3665771484375, + "KL/mean": -819.79736328125, + "KL/rejected_KL_mean": -1068.2281494140625, + "KL/std": 547.6316528320312, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0009246543631888926, + "fcm_dpo/delta": -0.06239794194698334, + "fcm_dpo/margin": 496.861572265625, + "fcm_dpo/q_t": 0.3960478901863098, + "grad_norm": 32.293460845947266, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.961599588394165, + "logits/rejected": -0.9833802580833435, + "logps/chosen": -633.7123413085938, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1165.168701171875, + "loss": 1.0679, + "margin_dpo/margin_mean": 496.8616027832031, + "margin_dpo/margin_std": 664.1732788085938, + "step": 670 + }, + { + "KL/chosen_KL_mean": -722.337646484375, + "KL/mean": -922.5555419921875, + "KL/rejected_KL_mean": -1122.7733154296875, + "KL/std": 550.896240234375, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.0009255155455321074, + "fcm_dpo/delta": 0.030080577358603477, + "fcm_dpo/margin": 400.4356689453125, + "fcm_dpo/q_t": 0.4147670865058899, + "grad_norm": 32.56404495239258, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.0954818725585938, + "logits/rejected": -1.1285096406936646, + "logps/chosen": -770.3377685546875, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1206.5926513671875, + "loss": 1.138, + "margin_dpo/margin_mean": 400.4356689453125, + "margin_dpo/margin_std": 662.7120971679688, + "step": 671 + }, + { + "KL/chosen_KL_mean": -776.2171630859375, + "KL/mean": -982.8839111328125, + "KL/rejected_KL_mean": -1189.5506591796875, + "KL/std": 652.0782470703125, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0009243548847734928, + "fcm_dpo/delta": 0.018623486161231995, + "fcm_dpo/margin": 413.3335876464844, + "fcm_dpo/q_t": 0.41586923599243164, + "grad_norm": 62.2723388671875, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.1082384586334229, + "logits/rejected": -1.1188971996307373, + "logps/chosen": -834.8004150390625, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1282.69091796875, + "loss": 1.144, + "margin_dpo/margin_mean": 413.33355712890625, + "margin_dpo/margin_std": 721.7978515625, + "step": 672 + }, + { + "KL/chosen_KL_mean": -705.1563720703125, + "KL/mean": -896.4896240234375, + "KL/rejected_KL_mean": -1087.822998046875, + "KL/std": 560.9285278320312, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0009321393445134163, + "fcm_dpo/delta": 0.04492665082216263, + "fcm_dpo/margin": 382.6665344238281, + "fcm_dpo/q_t": 0.41984766721725464, + "grad_norm": 33.99762725830078, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.0407583713531494, + "logits/rejected": -1.0472893714904785, + "logps/chosen": -751.8795166015625, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1173.119140625, + "loss": 1.1376, + "margin_dpo/margin_mean": 382.66656494140625, + "margin_dpo/margin_std": 633.066650390625, + "step": 673 + }, + { + "KL/chosen_KL_mean": -590.9700317382812, + "KL/mean": -841.1318359375, + "KL/rejected_KL_mean": -1091.293701171875, + "KL/std": 559.5699462890625, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0009278652141802013, + "fcm_dpo/delta": -0.06731262058019638, + "fcm_dpo/margin": 500.3235778808594, + "fcm_dpo/q_t": 0.397756427526474, + "grad_norm": 37.452877044677734, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.9867762327194214, + "logits/rejected": -1.0102487802505493, + "logps/chosen": -636.4155883789062, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1161.339599609375, + "loss": 1.0612, + "margin_dpo/margin_mean": 500.3235778808594, + "margin_dpo/margin_std": 674.74609375, + "step": 674 + }, + { + "KL/chosen_KL_mean": -663.5950927734375, + "KL/mean": -918.8189086914062, + "KL/rejected_KL_mean": -1174.042724609375, + "KL/std": 608.5978393554688, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0009064309997484088, + "fcm_dpo/delta": -0.0674276202917099, + "fcm_dpo/margin": 510.4475402832031, + "fcm_dpo/q_t": 0.3976379334926605, + "grad_norm": 27.067426681518555, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.0346945524215698, + "logits/rejected": -1.0489528179168701, + "logps/chosen": -707.7713623046875, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1248.1346435546875, + "loss": 1.0612, + "margin_dpo/margin_mean": 510.4475402832031, + "margin_dpo/margin_std": 655.10107421875, + "step": 675 + }, + { + "KL/chosen_KL_mean": -719.826171875, + "KL/mean": -958.3115234375, + "KL/rejected_KL_mean": -1196.7967529296875, + "KL/std": 579.0013427734375, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0009066574275493622, + "fcm_dpo/delta": -0.03390350937843323, + "fcm_dpo/margin": 476.9705810546875, + "fcm_dpo/q_t": 0.4010433554649353, + "grad_norm": 29.531612396240234, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.9864528179168701, + "logits/rejected": -1.0022345781326294, + "logps/chosen": -791.2247314453125, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1285.155517578125, + "loss": 1.0646, + "margin_dpo/margin_mean": 476.9705810546875, + "margin_dpo/margin_std": 579.742919921875, + "step": 676 + }, + { + "KL/chosen_KL_mean": -727.7244262695312, + "KL/mean": -963.2039794921875, + "KL/rejected_KL_mean": -1198.683349609375, + "KL/std": 599.0261840820312, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0008997026598080993, + "fcm_dpo/delta": -0.024826671928167343, + "fcm_dpo/margin": 470.95892333984375, + "fcm_dpo/q_t": 0.4085960388183594, + "grad_norm": 30.339242935180664, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.0695778131484985, + "logits/rejected": -1.069761037826538, + "logps/chosen": -784.2518310546875, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1276.909912109375, + "loss": 1.0973, + "margin_dpo/margin_mean": 470.9589538574219, + "margin_dpo/margin_std": 723.3491821289062, + "step": 677 + }, + { + "KL/chosen_KL_mean": -591.198974609375, + "KL/mean": -864.2771606445312, + "KL/rejected_KL_mean": -1137.3553466796875, + "KL/std": 648.10107421875, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0008910767501220107, + "fcm_dpo/delta": -0.09106584638357162, + "fcm_dpo/margin": 546.156494140625, + "fcm_dpo/q_t": 0.39156806468963623, + "grad_norm": 34.50929641723633, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.9937125444412231, + "logits/rejected": -1.0318535566329956, + "logps/chosen": -637.3333740234375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1217.9599609375, + "loss": 1.0526, + "margin_dpo/margin_mean": 546.156494140625, + "margin_dpo/margin_std": 735.871826171875, + "step": 678 + }, + { + "KL/chosen_KL_mean": -676.77587890625, + "KL/mean": -865.25341796875, + "KL/rejected_KL_mean": -1053.73095703125, + "KL/std": 519.2434692382812, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0008911984041333199, + "fcm_dpo/delta": 0.06633217632770538, + "fcm_dpo/margin": 376.95501708984375, + "fcm_dpo/q_t": 0.42380523681640625, + "grad_norm": 34.87147521972656, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -1.023957371711731, + "logits/rejected": -1.0142775774002075, + "logps/chosen": -727.07080078125, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1130.3291015625, + "loss": 1.1442, + "margin_dpo/margin_mean": 376.95501708984375, + "margin_dpo/margin_std": 597.436279296875, + "step": 679 + }, + { + "KL/chosen_KL_mean": -692.7845458984375, + "KL/mean": -938.5079345703125, + "KL/rejected_KL_mean": -1184.231201171875, + "KL/std": 669.75537109375, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0008862123358994722, + "fcm_dpo/delta": -0.03770776093006134, + "fcm_dpo/margin": 491.4466857910156, + "fcm_dpo/q_t": 0.3999880254268646, + "grad_norm": 44.6490592956543, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -1.009589433670044, + "logits/rejected": -1.0409965515136719, + "logps/chosen": -769.7003173828125, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1296.615966796875, + "loss": 1.0933, + "margin_dpo/margin_mean": 491.4466857910156, + "margin_dpo/margin_std": 728.8011474609375, + "step": 680 + }, + { + "KL/chosen_KL_mean": -688.2036743164062, + "KL/mean": -885.5780029296875, + "KL/rejected_KL_mean": -1082.952392578125, + "KL/std": 557.761962890625, + "epoch": 1.0, + "fcm_dpo/beta": 0.0009009799105115235, + "fcm_dpo/delta": 0.04453130066394806, + "fcm_dpo/margin": 394.74871826171875, + "fcm_dpo/q_t": 0.4194086790084839, + "grad_norm": 34.9099235534668, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.0720038414001465, + "logits/rejected": -1.090095043182373, + "logps/chosen": -749.1609497070312, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1171.5103759765625, + "loss": 1.1399, + "margin_dpo/margin_mean": 394.74871826171875, + "margin_dpo/margin_std": 612.0587768554688, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.0915097299301326, + "train_runtime": 1740.2795, + "train_samples_per_second": 25.052, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}