commit e58d4d9296390abe19da7d49e1abb12443e6be0f Author: ModelHub XC Date: Tue May 26 11:47:18 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-1 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..6a13fad --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-1 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-1 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..836fd92 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1280472663705745, + "train_runtime": 1739.8346, + "train_samples": 43598, + "train_samples_per_second": 25.059, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..14fdadc --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d38e8d75b6b17ef740e7f41812fed6b95687bb73bff6406eda020aa176434887 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..a4d6671 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dadeafd0a2188521c5a0ea4a63e5c1d98fdc9fc30a7a1fb7cadebe490fb0b9 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..b772069 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34229d67895c436cf2cdf731bb47031f181c1be178589c4529ee4214158a818d +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..b8089d9 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cba972bce7206109e04625e9da2ba9e65ba563ba6e9d396f0b89cc88d705810 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..a6378c5 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e510cbe376cadc477bc8d153c25f7d2784a31267743622c80ebdd28780ddbf11 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..0090a1b --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf666e70be49899ae523d69a9be8b3683adf1f570900230a44f1c1a09e0dbe5 +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..e56386c --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e02f25d4c9255f49bc79545d0941c7761b5bb308a0a53df08a02987b1f8b6b +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..836fd92 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1280472663705745, + "train_runtime": 1739.8346, + "train_samples": 43598, + "train_samples_per_second": 25.059, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c16f1df --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.52816009521484, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.20802307128906, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.016103744506835938, + "KL/mean": 0.019720733165740967, + "KL/rejected_KL_mean": 0.023336410522460938, + "KL/std": 0.25345832109451294, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007235169410705566, + "fcm_dpo/q_t": 0.5001815557479858, + "grad_norm": 70.94044494628906, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.48162591457366943, + "logits/rejected": -0.4422367215156555, + "logps/chosen": -60.965492248535156, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.64926147460938, + "loss": 1.3874, + "margin_dpo/margin_mean": -0.007234543561935425, + "margin_dpo/margin_std": 0.3761873245239258, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.007844924926757812, + "KL/mean": 0.011015832424163818, + "KL/rejected_KL_mean": 0.014186859130859375, + "KL/std": 0.27283304929733276, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.006340086460113525, + "fcm_dpo/q_t": 0.5001586675643921, + "grad_norm": 71.89739990234375, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.4688315987586975, + "logits/rejected": -0.4412044882774353, + "logps/chosen": -56.759864807128906, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.63291931152344, + "loss": 1.3873, + "margin_dpo/margin_mean": -0.00634080171585083, + "margin_dpo/margin_std": 0.4030381143093109, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.0037784576416015625, + "KL/mean": 0.03290402889251709, + "KL/rejected_KL_mean": 0.06203460693359375, + "KL/std": 0.3190087080001831, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.05825453996658325, + "fcm_dpo/q_t": 0.5014559030532837, + "grad_norm": 90.05491638183594, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5145796537399292, + "logits/rejected": -0.47089338302612305, + "logps/chosen": -53.85559844970703, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.08715057373047, + "loss": 1.3926, + "margin_dpo/margin_mean": -0.05825403332710266, + "margin_dpo/margin_std": 0.4133654236793518, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.008159637451171875, + "KL/mean": -0.009616628289222717, + "KL/rejected_KL_mean": -0.011074066162109375, + "KL/std": 0.2630811631679535, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0029135048389434814, + "fcm_dpo/q_t": 0.49992650747299194, + "grad_norm": 91.91436004638672, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5270358920097351, + "logits/rejected": -0.48671644926071167, + "logps/chosen": -63.01564407348633, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.65641784667969, + "loss": 1.3864, + "margin_dpo/margin_mean": 0.002912759780883789, + "margin_dpo/margin_std": 0.37860625982284546, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.018707275390625, + "KL/mean": 0.022922277450561523, + "KL/rejected_KL_mean": 0.027141571044921875, + "KL/std": 0.28703397512435913, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.008430540561676025, + "fcm_dpo/q_t": 0.5002114176750183, + "grad_norm": 82.6740493774414, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.498199999332428, + "logits/rejected": -0.4643540382385254, + "logps/chosen": -57.75611114501953, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.89344787597656, + "loss": 1.3876, + "margin_dpo/margin_mean": -0.008429855108261108, + "margin_dpo/margin_std": 0.42599087953567505, + "step": 7 + }, + { + "KL/chosen_KL_mean": -0.012296676635742188, + "KL/mean": 0.0035399198532104492, + "KL/rejected_KL_mean": 0.019378662109375, + "KL/std": 0.2982986569404602, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.031677067279815674, + "fcm_dpo/q_t": 0.5007911920547485, + "grad_norm": 79.09713745117188, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5000207424163818, + "logits/rejected": -0.4732367694377899, + "logps/chosen": -58.72833251953125, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.29204559326172, + "loss": 1.3899, + "margin_dpo/margin_mean": -0.03167632222175598, + "margin_dpo/margin_std": 0.404108464717865, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.021236419677734375, + "KL/mean": 0.0014689713716506958, + "KL/rejected_KL_mean": -0.018299102783203125, + "KL/std": 0.270100474357605, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.039538174867630005, + "fcm_dpo/q_t": 0.4990116357803345, + "grad_norm": 84.92678833007812, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.4937548339366913, + "logits/rejected": -0.4469431936740875, + "logps/chosen": -69.84561157226562, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.62096405029297, + "loss": 1.3827, + "margin_dpo/margin_mean": 0.03953742980957031, + "margin_dpo/margin_std": 0.35995495319366455, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.003574371337890625, + "KL/mean": -0.0032436102628707886, + "KL/rejected_KL_mean": -0.010059356689453125, + "KL/std": 0.25196051597595215, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.013635605573654175, + "fcm_dpo/q_t": 0.49965864419937134, + "grad_norm": 70.92430114746094, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.5009649991989136, + "logits/rejected": -0.4579939842224121, + "logps/chosen": -48.35411071777344, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.38212585449219, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.013636231422424316, + "margin_dpo/margin_std": 0.36122623085975647, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.005367279052734375, + "KL/mean": -0.006761014461517334, + "KL/rejected_KL_mean": -0.018890380859375, + "KL/std": 0.24701017141342163, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.024251699447631836, + "fcm_dpo/q_t": 0.49939417839050293, + "grad_norm": 68.93864440917969, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.45688319206237793, + "logits/rejected": -0.4317648708820343, + "logps/chosen": -53.01148986816406, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.79927062988281, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.024251878261566162, + "margin_dpo/margin_std": 0.3476247489452362, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.05808830261230469, + "KL/mean": -0.053139448165893555, + "KL/rejected_KL_mean": -0.048183441162109375, + "KL/std": 0.2798803448677063, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.009897112846374512, + "fcm_dpo/q_t": 0.5002454519271851, + "grad_norm": 89.550048828125, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5406717658042908, + "logits/rejected": -0.5045890808105469, + "logps/chosen": -61.863525390625, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.90645599365234, + "loss": 1.3877, + "margin_dpo/margin_mean": -0.009897887706756592, + "margin_dpo/margin_std": 0.40574026107788086, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.030796051025390625, + "KL/mean": 0.014434963464736938, + "KL/rejected_KL_mean": -0.00193023681640625, + "KL/std": 0.294820100069046, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.032728374004364014, + "fcm_dpo/q_t": 0.49918168783187866, + "grad_norm": 79.24481201171875, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.4753603935241699, + "logits/rejected": -0.44688206911087036, + "logps/chosen": -64.22956085205078, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.20501708984375, + "loss": 1.3835, + "margin_dpo/margin_mean": 0.03272801637649536, + "margin_dpo/margin_std": 0.4066898226737976, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.05864906311035156, + "KL/mean": -0.018393635749816895, + "KL/rejected_KL_mean": 0.021862030029296875, + "KL/std": 0.29221126437187195, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0805087685585022, + "fcm_dpo/q_t": 0.5020102858543396, + "grad_norm": 85.43807983398438, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.4688841998577118, + "logits/rejected": -0.4292323887348175, + "logps/chosen": -58.168861389160156, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.02522277832031, + "loss": 1.3949, + "margin_dpo/margin_mean": -0.08050811290740967, + "margin_dpo/margin_std": 0.4493297040462494, + "step": 14 + }, + { + "KL/chosen_KL_mean": 0.013751983642578125, + "KL/mean": -0.004344776272773743, + "KL/rejected_KL_mean": -0.022441864013671875, + "KL/std": 0.2262694537639618, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.036195188760757446, + "fcm_dpo/q_t": 0.49909502267837524, + "grad_norm": 64.15103912353516, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.5063309669494629, + "logits/rejected": -0.48831403255462646, + "logps/chosen": -56.953155517578125, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.83108520507812, + "loss": 1.3829, + "margin_dpo/margin_mean": 0.036195307970047, + "margin_dpo/margin_std": 0.31367939710617065, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.02686309814453125, + "KL/mean": -0.023277834057807922, + "KL/rejected_KL_mean": -0.01969146728515625, + "KL/std": 0.28893446922302246, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007169216871261597, + "fcm_dpo/q_t": 0.5001786947250366, + "grad_norm": 83.99118041992188, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5097619891166687, + "logits/rejected": -0.46720778942108154, + "logps/chosen": -61.766754150390625, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.38916015625, + "loss": 1.3874, + "margin_dpo/margin_mean": -0.007169753313064575, + "margin_dpo/margin_std": 0.4101220369338989, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.011844635009765625, + "KL/mean": -0.012153714895248413, + "KL/rejected_KL_mean": -0.03615570068359375, + "KL/std": 0.26074981689453125, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.047999799251556396, + "fcm_dpo/q_t": 0.4987994134426117, + "grad_norm": 78.72090148925781, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.5130709409713745, + "logits/rejected": -0.47745299339294434, + "logps/chosen": -67.698486328125, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.41481018066406, + "loss": 1.3818, + "margin_dpo/margin_mean": 0.04799976944923401, + "margin_dpo/margin_std": 0.34855252504348755, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.01177215576171875, + "KL/mean": -0.058126211166381836, + "KL/rejected_KL_mean": -0.10448455810546875, + "KL/std": 0.24388915300369263, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.092709481716156, + "fcm_dpo/q_t": 0.49768269062042236, + "grad_norm": 81.854248046875, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.49614471197128296, + "logits/rejected": -0.4409538507461548, + "logps/chosen": -47.75126266479492, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.57676696777344, + "loss": 1.3773, + "margin_dpo/margin_mean": 0.09270939230918884, + "margin_dpo/margin_std": 0.3233555257320404, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.007350921630859375, + "KL/mean": -0.03270593285560608, + "KL/rejected_KL_mean": -0.07275772094726562, + "KL/std": 0.22124189138412476, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08010363578796387, + "fcm_dpo/q_t": 0.4979983866214752, + "grad_norm": 73.39251708984375, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.5021491050720215, + "logits/rejected": -0.4532221555709839, + "logps/chosen": -70.19801330566406, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.83035278320312, + "loss": 1.3786, + "margin_dpo/margin_mean": 0.08010467886924744, + "margin_dpo/margin_std": 0.34056466817855835, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.010915756225585938, + "KL/mean": -0.03122629225254059, + "KL/rejected_KL_mean": -0.07336044311523438, + "KL/std": 0.26908406615257263, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08428379893302917, + "fcm_dpo/q_t": 0.4978940486907959, + "grad_norm": 73.79589080810547, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5578715801239014, + "logits/rejected": -0.5020963549613953, + "logps/chosen": -50.792327880859375, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.89671325683594, + "loss": 1.3783, + "margin_dpo/margin_mean": 0.08428427577018738, + "margin_dpo/margin_std": 0.3846767842769623, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.016414642333984375, + "KL/mean": -0.05401710420846939, + "KL/rejected_KL_mean": -0.12445068359375, + "KL/std": 0.31130141019821167, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.14086376130580902, + "fcm_dpo/q_t": 0.49647945165634155, + "grad_norm": 77.40738677978516, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.4897395968437195, + "logits/rejected": -0.4655163288116455, + "logps/chosen": -50.046600341796875, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.99324035644531, + "loss": 1.3726, + "margin_dpo/margin_mean": 0.14086447656154633, + "margin_dpo/margin_std": 0.3929217755794525, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.07324600219726562, + "KL/mean": -0.054158031940460205, + "KL/rejected_KL_mean": -0.18156051635742188, + "KL/std": 0.3565206825733185, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2548089623451233, + "fcm_dpo/q_t": 0.4936355948448181, + "grad_norm": 83.94691467285156, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.4774931073188782, + "logits/rejected": -0.43337398767471313, + "logps/chosen": -58.98439025878906, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.68623352050781, + "loss": 1.3617, + "margin_dpo/margin_mean": 0.2548080384731293, + "margin_dpo/margin_std": 0.523194432258606, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.026395797729492188, + "KL/mean": -0.0561884343624115, + "KL/rejected_KL_mean": -0.13876724243164062, + "KL/std": 0.30823904275894165, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1651589274406433, + "fcm_dpo/q_t": 0.49587440490722656, + "grad_norm": 80.6090087890625, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5116769671440125, + "logits/rejected": -0.4903678894042969, + "logps/chosen": -60.051300048828125, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.2783203125, + "loss": 1.3703, + "margin_dpo/margin_mean": 0.16515937447547913, + "margin_dpo/margin_std": 0.4328920841217041, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.053745269775390625, + "KL/mean": -0.060146935284137726, + "KL/rejected_KL_mean": -0.174041748046875, + "KL/std": 0.277465283870697, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22778569161891937, + "fcm_dpo/q_t": 0.49430739879608154, + "grad_norm": 82.69357299804688, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5428138375282288, + "logits/rejected": -0.525646448135376, + "logps/chosen": -44.23728942871094, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.29925537109375, + "loss": 1.364, + "margin_dpo/margin_mean": 0.22778579592704773, + "margin_dpo/margin_std": 0.36717647314071655, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.017560958862304688, + "KL/mean": -0.10330188274383545, + "KL/rejected_KL_mean": -0.22416305541992188, + "KL/std": 0.36914387345314026, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24172252416610718, + "fcm_dpo/q_t": 0.4939602315425873, + "grad_norm": 73.93270111083984, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.5116393566131592, + "logits/rejected": -0.48214203119277954, + "logps/chosen": -52.519493103027344, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.56635284423828, + "loss": 1.3628, + "margin_dpo/margin_mean": 0.24172285199165344, + "margin_dpo/margin_std": 0.4416242837905884, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.13255691528320312, + "KL/mean": -0.08234043419361115, + "KL/rejected_KL_mean": -0.2972412109375, + "KL/std": 0.4482305645942688, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.42980238795280457, + "fcm_dpo/q_t": 0.48926833271980286, + "grad_norm": 86.75181579589844, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5427736043930054, + "logits/rejected": -0.5113131999969482, + "logps/chosen": -53.790252685546875, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.65695190429688, + "loss": 1.3445, + "margin_dpo/margin_mean": 0.42980268597602844, + "margin_dpo/margin_std": 0.5426281690597534, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.12249946594238281, + "KL/mean": -0.1331784576177597, + "KL/rejected_KL_mean": -0.3888587951660156, + "KL/std": 0.4570690989494324, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5113544464111328, + "fcm_dpo/q_t": 0.4872331917285919, + "grad_norm": 93.51718139648438, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5056776404380798, + "logits/rejected": -0.4691791832447052, + "logps/chosen": -42.776031494140625, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.1130599975586, + "loss": 1.3366, + "margin_dpo/margin_mean": 0.5113544464111328, + "margin_dpo/margin_std": 0.5590527057647705, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.03354072570800781, + "KL/mean": -0.11374713480472565, + "KL/rejected_KL_mean": -0.2610359191894531, + "KL/std": 0.41249018907546997, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.29457351565361023, + "fcm_dpo/q_t": 0.49264103174209595, + "grad_norm": 75.92487335205078, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.49452316761016846, + "logits/rejected": -0.4381081163883209, + "logps/chosen": -60.5229606628418, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.66215515136719, + "loss": 1.3579, + "margin_dpo/margin_mean": 0.29457390308380127, + "margin_dpo/margin_std": 0.5741066336631775, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.06393051147460938, + "KL/mean": -0.18335245549678802, + "KL/rejected_KL_mean": -0.430633544921875, + "KL/std": 0.4628360867500305, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4945622384548187, + "fcm_dpo/q_t": 0.4876513183116913, + "grad_norm": 91.02857971191406, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5597679615020752, + "logits/rejected": -0.5140695571899414, + "logps/chosen": -57.74385452270508, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.82498168945312, + "loss": 1.3383, + "margin_dpo/margin_mean": 0.4945625960826874, + "margin_dpo/margin_std": 0.5654176473617554, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.1397686004638672, + "KL/mean": -0.20081256330013275, + "KL/rejected_KL_mean": -0.5413970947265625, + "KL/std": 0.5910484194755554, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6811658143997192, + "fcm_dpo/q_t": 0.48299551010131836, + "grad_norm": 87.69357299804688, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.4750695824623108, + "logits/rejected": -0.4435637593269348, + "logps/chosen": -52.43760299682617, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.03060150146484, + "loss": 1.3203, + "margin_dpo/margin_mean": 0.6811659336090088, + "margin_dpo/margin_std": 0.6095134019851685, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.1102294921875, + "KL/mean": -0.13631178438663483, + "KL/rejected_KL_mean": -0.3828544616699219, + "KL/std": 0.5659123063087463, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.49308571219444275, + "fcm_dpo/q_t": 0.4877015948295593, + "grad_norm": 67.67135620117188, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.5170990228652954, + "logits/rejected": -0.4736338257789612, + "logps/chosen": -63.696693420410156, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.27685546875, + "loss": 1.3388, + "margin_dpo/margin_mean": 0.49308544397354126, + "margin_dpo/margin_std": 0.6761853694915771, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.21260833740234375, + "KL/mean": -0.19022028148174286, + "KL/rejected_KL_mean": -0.5930519104003906, + "KL/std": 0.7161545753479004, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8056564331054688, + "fcm_dpo/q_t": 0.47991523146629333, + "grad_norm": 83.33721160888672, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.504935085773468, + "logits/rejected": -0.46319282054901123, + "logps/chosen": -62.52691650390625, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.91055297851562, + "loss": 1.3091, + "margin_dpo/margin_mean": 0.8056566715240479, + "margin_dpo/margin_std": 0.8240557909011841, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.09860420227050781, + "KL/mean": -0.18287688493728638, + "KL/rejected_KL_mean": -0.4643592834472656, + "KL/std": 0.5310133099555969, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5629608035087585, + "fcm_dpo/q_t": 0.48594412207603455, + "grad_norm": 72.46797180175781, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.5104891657829285, + "logits/rejected": -0.4849121570587158, + "logps/chosen": -53.16236877441406, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.34949493408203, + "loss": 1.3317, + "margin_dpo/margin_mean": 0.5629609823226929, + "margin_dpo/margin_std": 0.5864159464836121, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.04570198059082031, + "KL/mean": -0.32518571615219116, + "KL/rejected_KL_mean": -0.696075439453125, + "KL/std": 0.7097070813179016, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7417768239974976, + "fcm_dpo/q_t": 0.48150891065597534, + "grad_norm": 77.95159912109375, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.4825457036495209, + "logits/rejected": -0.4650344252586365, + "logps/chosen": -50.77162551879883, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.617919921875, + "loss": 1.3155, + "margin_dpo/margin_mean": 0.7417754530906677, + "margin_dpo/margin_std": 0.8860844969749451, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.15726661682128906, + "KL/mean": -0.44504478573799133, + "KL/rejected_KL_mean": -1.0473518371582031, + "KL/std": 0.9632636308670044, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2046229839324951, + "fcm_dpo/q_t": 0.47004756331443787, + "grad_norm": 83.60721588134766, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5268377065658569, + "logits/rejected": -0.49007338285446167, + "logps/chosen": -50.867225646972656, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.87178039550781, + "loss": 1.2721, + "margin_dpo/margin_mean": 1.204622745513916, + "margin_dpo/margin_std": 1.0357599258422852, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.02988433837890625, + "KL/mean": -0.5372347831726074, + "KL/rejected_KL_mean": -1.1043548583984375, + "KL/std": 1.094662070274353, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1342408657073975, + "fcm_dpo/q_t": 0.47180622816085815, + "grad_norm": 72.68156433105469, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5595717430114746, + "logits/rejected": -0.523262619972229, + "logps/chosen": -51.96160888671875, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.14498138427734, + "loss": 1.2802, + "margin_dpo/margin_mean": 1.1342405080795288, + "margin_dpo/margin_std": 1.2309517860412598, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.023458480834960938, + "KL/mean": -0.4883181154727936, + "KL/rejected_KL_mean": -1.0000991821289062, + "KL/std": 1.0986820459365845, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0235590934753418, + "fcm_dpo/q_t": 0.4746357798576355, + "grad_norm": 61.56761932373047, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5261737108230591, + "logits/rejected": -0.4837297797203064, + "logps/chosen": -62.78364944458008, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.89517211914062, + "loss": 1.2918, + "margin_dpo/margin_mean": 1.0235590934753418, + "margin_dpo/margin_std": 1.4123997688293457, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1512908935546875, + "KL/mean": -0.5362459421157837, + "KL/rejected_KL_mean": -1.2237815856933594, + "KL/std": 1.3535866737365723, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3750734329223633, + "fcm_dpo/q_t": 0.4660550653934479, + "grad_norm": 69.93470001220703, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5330358743667603, + "logits/rejected": -0.5002619028091431, + "logps/chosen": -48.239227294921875, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.13622283935547, + "loss": 1.2601, + "margin_dpo/margin_mean": 1.3750736713409424, + "margin_dpo/margin_std": 1.628260850906372, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.08315658569335938, + "KL/mean": -0.7190086841583252, + "KL/rejected_KL_mean": -1.5211677551269531, + "KL/std": 1.2815454006195068, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6043219566345215, + "fcm_dpo/q_t": 0.46017903089523315, + "grad_norm": 74.16421508789062, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5571820735931396, + "logits/rejected": -0.5172927379608154, + "logps/chosen": -50.66731643676758, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.09068298339844, + "loss": 1.2371, + "margin_dpo/margin_mean": 1.6043217182159424, + "margin_dpo/margin_std": 1.3714519739151, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.18222999572753906, + "KL/mean": -0.5860625505447388, + "KL/rejected_KL_mean": -1.3543624877929688, + "KL/std": 1.4266393184661865, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5365939140319824, + "fcm_dpo/q_t": 0.46209070086479187, + "grad_norm": 60.399505615234375, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5290929079055786, + "logits/rejected": -0.4998553693294525, + "logps/chosen": -57.80283737182617, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.65443420410156, + "loss": 1.246, + "margin_dpo/margin_mean": 1.5365946292877197, + "margin_dpo/margin_std": 1.694861650466919, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.00180816650390625, + "KL/mean": -0.9451515078544617, + "KL/rejected_KL_mean": -1.8921089172363281, + "KL/std": 1.8964862823486328, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8939173221588135, + "fcm_dpo/q_t": 0.4534837007522583, + "grad_norm": 68.17810821533203, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5442007184028625, + "logits/rejected": -0.5080921649932861, + "logps/chosen": -62.694007873535156, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.91563415527344, + "loss": 1.217, + "margin_dpo/margin_mean": 1.8939182758331299, + "margin_dpo/margin_std": 2.07293701171875, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.20568084716796875, + "KL/mean": -1.066096544265747, + "KL/rejected_KL_mean": -2.337871551513672, + "KL/std": 2.129526138305664, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.12230241298675537, + "fcm_dpo/delta": 0.12432946264743805, + "fcm_dpo/margin": 2.5435502529144287, + "fcm_dpo/q_t": 0.43291333317756653, + "grad_norm": 92.2998046875, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5354119539260864, + "logits/rejected": -0.4888347387313843, + "logps/chosen": -58.760746002197266, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.24624633789062, + "loss": 1.119, + "margin_dpo/margin_mean": 2.5435495376586914, + "margin_dpo/margin_std": 2.3769054412841797, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5492343902587891, + "KL/mean": -0.6893892288208008, + "KL/rejected_KL_mean": -1.9280166625976562, + "KL/std": 1.8261648416519165, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.14197269082069397, + "fcm_dpo/delta": 0.07470498234033585, + "fcm_dpo/margin": 2.477250099182129, + "fcm_dpo/q_t": 0.42087799310684204, + "grad_norm": 93.16883087158203, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5261520147323608, + "logits/rejected": -0.5005497932434082, + "logps/chosen": -53.60676193237305, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.408203125, + "loss": 1.0844, + "margin_dpo/margin_mean": 2.477250099182129, + "margin_dpo/margin_std": 1.9302922487258911, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.22637557983398438, + "KL/mean": -1.1899750232696533, + "KL/rejected_KL_mean": -2.6063270568847656, + "KL/std": 2.267411231994629, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.14421430230140686, + "fcm_dpo/delta": -0.01492190733551979, + "fcm_dpo/margin": 2.8327009677886963, + "fcm_dpo/q_t": 0.40067583322525024, + "grad_norm": 101.48235321044922, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4636678099632263, + "logits/rejected": -0.44394707679748535, + "logps/chosen": -49.85212326049805, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.39009094238281, + "loss": 1.0429, + "margin_dpo/margin_mean": 2.8327014446258545, + "margin_dpo/margin_std": 2.2093987464904785, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.19671249389648438, + "KL/mean": -0.9757533073425293, + "KL/rejected_KL_mean": -2.1482200622558594, + "KL/std": 2.1102242469787598, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.15825220942497253, + "fcm_dpo/delta": 0.041776977479457855, + "fcm_dpo/margin": 2.344935417175293, + "fcm_dpo/q_t": 0.4162585139274597, + "grad_norm": 85.96018981933594, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.5120048522949219, + "logits/rejected": -0.5001665353775024, + "logps/chosen": -48.21821594238281, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.08464813232422, + "loss": 1.0898, + "margin_dpo/margin_mean": 2.344935417175293, + "margin_dpo/margin_std": 2.62362003326416, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.17397499084472656, + "KL/mean": -1.2946666479110718, + "KL/rejected_KL_mean": -2.7633094787597656, + "KL/std": 2.6588382720947266, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.14704476296901703, + "fcm_dpo/delta": -0.05587401241064072, + "fcm_dpo/margin": 2.9372873306274414, + "fcm_dpo/q_t": 0.3960358500480652, + "grad_norm": 88.10020446777344, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.517956018447876, + "logits/rejected": -0.46634984016418457, + "logps/chosen": -55.8254508972168, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.4158935546875, + "loss": 1.0542, + "margin_dpo/margin_mean": 2.9372870922088623, + "margin_dpo/margin_std": 3.269395351409912, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.4055042266845703, + "KL/mean": -1.0942176580429077, + "KL/rejected_KL_mean": -2.5939369201660156, + "KL/std": 2.6453235149383545, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.13928335905075073, + "fcm_dpo/delta": -0.032578371465206146, + "fcm_dpo/margin": 2.9994425773620605, + "fcm_dpo/q_t": 0.3979858160018921, + "grad_norm": 82.3074951171875, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5774295330047607, + "logits/rejected": -0.5251759886741638, + "logps/chosen": -57.52057647705078, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.27314758300781, + "loss": 1.0483, + "margin_dpo/margin_mean": 2.999443769454956, + "margin_dpo/margin_std": 2.69224214553833, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.05373954772949219, + "KL/mean": -1.51072359085083, + "KL/rejected_KL_mean": -3.075183868408203, + "KL/std": 2.700470447540283, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.12641848623752594, + "fcm_dpo/delta": 0.0021915659308433533, + "fcm_dpo/margin": 3.1289291381835938, + "fcm_dpo/q_t": 0.406181663274765, + "grad_norm": 87.43419647216797, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5810247659683228, + "logits/rejected": -0.5221822261810303, + "logps/chosen": -57.134334564208984, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.09178161621094, + "loss": 1.0689, + "margin_dpo/margin_mean": 3.1289291381835938, + "margin_dpo/margin_std": 2.9688820838928223, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.3045005798339844, + "KL/mean": -1.5968005657196045, + "KL/rejected_KL_mean": -3.498096466064453, + "KL/std": 3.415726661682129, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.11936050653457642, + "fcm_dpo/delta": -0.0978364571928978, + "fcm_dpo/margin": 3.8025965690612793, + "fcm_dpo/q_t": 0.38723063468933105, + "grad_norm": 69.9267578125, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5347216725349426, + "logits/rejected": -0.4745851755142212, + "logps/chosen": -61.380767822265625, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.26557159423828, + "loss": 1.0376, + "margin_dpo/margin_mean": 3.8025965690612793, + "margin_dpo/margin_std": 3.9496779441833496, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.15890121459960938, + "KL/mean": -2.2356748580932617, + "KL/rejected_KL_mean": -4.31243896484375, + "KL/std": 3.5314018726348877, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10124868154525757, + "fcm_dpo/delta": -0.03730197250843048, + "fcm_dpo/margin": 4.153538227081299, + "fcm_dpo/q_t": 0.3976140022277832, + "grad_norm": 63.207149505615234, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5480292439460754, + "logits/rejected": -0.512259304523468, + "logps/chosen": -58.88304138183594, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.67058563232422, + "loss": 1.0489, + "margin_dpo/margin_mean": 4.153538703918457, + "margin_dpo/margin_std": 4.076920986175537, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2769756317138672, + "KL/mean": -2.2801334857940674, + "KL/rejected_KL_mean": -4.283294677734375, + "KL/std": 4.116324424743652, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.09770938754081726, + "fcm_dpo/delta": 0.00890747457742691, + "fcm_dpo/margin": 4.006317138671875, + "fcm_dpo/q_t": 0.4107528030872345, + "grad_norm": 52.39238357543945, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.539941132068634, + "logits/rejected": -0.5076016187667847, + "logps/chosen": -61.65064239501953, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.28529357910156, + "loss": 1.0914, + "margin_dpo/margin_mean": 4.006316661834717, + "margin_dpo/margin_std": 5.059848308563232, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.4057025909423828, + "KL/mean": -2.6870739459991455, + "KL/rejected_KL_mean": -5.779850006103516, + "KL/std": 4.8188886642456055, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.077643021941185, + "fcm_dpo/delta": -0.17084355652332306, + "fcm_dpo/margin": 6.185553073883057, + "fcm_dpo/q_t": 0.3687407672405243, + "grad_norm": 52.488014221191406, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5524188876152039, + "logits/rejected": -0.4968685805797577, + "logps/chosen": -51.93165588378906, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.75376892089844, + "loss": 0.9997, + "margin_dpo/margin_mean": 6.185553073883057, + "margin_dpo/margin_std": 5.159676551818848, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.19318580627441406, + "KL/mean": -3.297855854034424, + "KL/rejected_KL_mean": -6.402523040771484, + "KL/std": 5.35381555557251, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.07053211331367493, + "fcm_dpo/delta": -0.07264071702957153, + "fcm_dpo/margin": 6.209340572357178, + "fcm_dpo/q_t": 0.3935110867023468, + "grad_norm": 48.63420104980469, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6061801910400391, + "logits/rejected": -0.5843982696533203, + "logps/chosen": -53.507835388183594, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.18611907958984, + "loss": 1.0485, + "margin_dpo/margin_mean": 6.209340572357178, + "margin_dpo/margin_std": 6.154953479766846, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3580665588378906, + "KL/mean": -3.091987133026123, + "KL/rejected_KL_mean": -5.825908660888672, + "KL/std": 5.11324405670166, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.06830751895904541, + "fcm_dpo/delta": 0.039879053831100464, + "fcm_dpo/margin": 5.467842102050781, + "fcm_dpo/q_t": 0.41404271125793457, + "grad_norm": 49.40464782714844, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5767498016357422, + "logits/rejected": -0.5225586295127869, + "logps/chosen": -51.046730041503906, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.54130554199219, + "loss": 1.0757, + "margin_dpo/margin_mean": 5.467841148376465, + "margin_dpo/margin_std": 5.10695743560791, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9405612945556641, + "KL/mean": -4.179176330566406, + "KL/rejected_KL_mean": -7.417789459228516, + "KL/std": 6.182194232940674, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.06581026315689087, + "fcm_dpo/delta": -0.04618923366069794, + "fcm_dpo/margin": 6.477225303649902, + "fcm_dpo/q_t": 0.3977408707141876, + "grad_norm": 41.99592971801758, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6314051151275635, + "logits/rejected": -0.5677323341369629, + "logps/chosen": -63.55579376220703, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.41129302978516, + "loss": 1.0627, + "margin_dpo/margin_mean": 6.4772257804870605, + "margin_dpo/margin_std": 7.436443328857422, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.7300090789794922, + "KL/mean": -3.821871280670166, + "KL/rejected_KL_mean": -6.913738250732422, + "KL/std": 6.0643815994262695, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.06379474699497223, + "fcm_dpo/delta": 0.00866011157631874, + "fcm_dpo/margin": 6.183729648590088, + "fcm_dpo/q_t": 0.4088730216026306, + "grad_norm": 37.987911224365234, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.5912456512451172, + "logits/rejected": -0.5477725267410278, + "logps/chosen": -58.66273498535156, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.08818054199219, + "loss": 1.0816, + "margin_dpo/margin_mean": 6.183730602264404, + "margin_dpo/margin_std": 7.2952046394348145, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7435302734375, + "KL/mean": -4.197587966918945, + "KL/rejected_KL_mean": -7.651645660400391, + "KL/std": 5.97504997253418, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.061574921011924744, + "fcm_dpo/delta": -0.04527352750301361, + "fcm_dpo/margin": 6.908114910125732, + "fcm_dpo/q_t": 0.3953145742416382, + "grad_norm": 41.76232147216797, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5989680290222168, + "logits/rejected": -0.571702241897583, + "logps/chosen": -71.23881530761719, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.21710205078125, + "loss": 1.0479, + "margin_dpo/margin_mean": 6.908115386962891, + "margin_dpo/margin_std": 6.625408172607422, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.0145072937011719, + "KL/mean": -4.882469177246094, + "KL/rejected_KL_mean": -8.75042724609375, + "KL/std": 6.831453323364258, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.0555315725505352, + "fcm_dpo/delta": -0.05244975909590721, + "fcm_dpo/margin": 7.735917568206787, + "fcm_dpo/q_t": 0.3955305814743042, + "grad_norm": 40.50571060180664, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.5726555585861206, + "logits/rejected": -0.49422115087509155, + "logps/chosen": -63.14744567871094, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.36772155761719, + "loss": 1.0532, + "margin_dpo/margin_mean": 7.7359185218811035, + "margin_dpo/margin_std": 8.176453590393066, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.488332748413086, + "KL/mean": -5.7610859870910645, + "KL/rejected_KL_mean": -10.033843994140625, + "KL/std": 7.584994792938232, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.047585196793079376, + "fcm_dpo/delta": -0.022728845477104187, + "fcm_dpo/margin": 8.545510292053223, + "fcm_dpo/q_t": 0.40144288539886475, + "grad_norm": 37.327362060546875, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6131638288497925, + "logits/rejected": -0.572055459022522, + "logps/chosen": -53.420860290527344, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.9190444946289, + "loss": 1.0661, + "margin_dpo/margin_mean": 8.545509338378906, + "margin_dpo/margin_std": 8.80247688293457, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.7225379943847656, + "KL/mean": -6.128490447998047, + "KL/rejected_KL_mean": -9.534439086914062, + "KL/std": 6.866521835327148, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.053369902074337006, + "fcm_dpo/delta": 0.05756625533103943, + "fcm_dpo/margin": 6.811899185180664, + "fcm_dpo/q_t": 0.4179559648036957, + "grad_norm": 40.32707214355469, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.6027133464813232, + "logits/rejected": -0.5420011878013611, + "logps/chosen": -63.66472625732422, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.92784118652344, + "loss": 1.0996, + "margin_dpo/margin_mean": 6.811899185180664, + "margin_dpo/margin_std": 7.766169548034668, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.580484390258789, + "KL/mean": -5.808887481689453, + "KL/rejected_KL_mean": -10.03729248046875, + "KL/std": 9.263717651367188, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.05006622523069382, + "fcm_dpo/delta": -0.04818345606327057, + "fcm_dpo/margin": 8.456808090209961, + "fcm_dpo/q_t": 0.40202924609184265, + "grad_norm": 35.04801940917969, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6034557819366455, + "logits/rejected": -0.5694935917854309, + "logps/chosen": -62.21400833129883, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.88978576660156, + "loss": 1.0795, + "margin_dpo/margin_mean": 8.456808090209961, + "margin_dpo/margin_std": 11.565231323242188, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.7315292358398438, + "KL/mean": -4.907068729400635, + "KL/rejected_KL_mean": -8.08260726928711, + "KL/std": 6.785726070404053, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.057009097188711166, + "fcm_dpo/delta": 0.05251770839095116, + "fcm_dpo/margin": 6.351075649261475, + "fcm_dpo/q_t": 0.4195292592048645, + "grad_norm": 36.52075958251953, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.6032343506813049, + "logits/rejected": -0.5685884952545166, + "logps/chosen": -57.882301330566406, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.6488037109375, + "loss": 1.1108, + "margin_dpo/margin_mean": 6.351076126098633, + "margin_dpo/margin_std": 8.331364631652832, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.8989200592041016, + "KL/mean": -7.218653678894043, + "KL/rejected_KL_mean": -11.538387298583984, + "KL/std": 8.756481170654297, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.04904904216527939, + "fcm_dpo/delta": -0.0558142215013504, + "fcm_dpo/margin": 8.639472961425781, + "fcm_dpo/q_t": 0.3941209316253662, + "grad_norm": 36.34617614746094, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.608661949634552, + "logits/rejected": -0.5645633935928345, + "logps/chosen": -76.04631042480469, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -109.14845275878906, + "loss": 1.0578, + "margin_dpo/margin_mean": 8.639472007751465, + "margin_dpo/margin_std": 9.21734619140625, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.1949291229248047, + "KL/mean": -6.278130054473877, + "KL/rejected_KL_mean": -11.361335754394531, + "KL/std": 9.37601089477539, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.04156487435102463, + "fcm_dpo/delta": -0.07142579555511475, + "fcm_dpo/margin": 10.166400909423828, + "fcm_dpo/q_t": 0.39335623383522034, + "grad_norm": 31.952661514282227, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.6194932460784912, + "logits/rejected": -0.5888575911521912, + "logps/chosen": -55.19352722167969, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.89154052734375, + "loss": 1.0654, + "margin_dpo/margin_mean": 10.166400909423828, + "margin_dpo/margin_std": 11.463658332824707, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.5524654388427734, + "KL/mean": -8.784185409545898, + "KL/rejected_KL_mean": -14.01590347290039, + "KL/std": 10.731781005859375, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.03978603705763817, + "fcm_dpo/delta": -0.03537318855524063, + "fcm_dpo/margin": 10.463440895080566, + "fcm_dpo/q_t": 0.39846062660217285, + "grad_norm": 32.09275436401367, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6814702153205872, + "logits/rejected": -0.6694004535675049, + "logps/chosen": -68.38846588134766, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.96236419677734, + "loss": 1.0664, + "margin_dpo/margin_mean": 10.46343994140625, + "margin_dpo/margin_std": 11.84306812286377, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.371671676635742, + "KL/mean": -8.132331848144531, + "KL/rejected_KL_mean": -12.892993927001953, + "KL/std": 9.663976669311523, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.04018988832831383, + "fcm_dpo/delta": 0.02319152280688286, + "fcm_dpo/margin": 9.521313667297363, + "fcm_dpo/q_t": 0.4129948616027832, + "grad_norm": 28.723054885864258, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6573896408081055, + "logits/rejected": -0.6258237957954407, + "logps/chosen": -54.8151969909668, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -88.529296875, + "loss": 1.0948, + "margin_dpo/margin_mean": 9.521312713623047, + "margin_dpo/margin_std": 11.660235404968262, + "step": 66 + }, + { + "KL/chosen_KL_mean": -3.1228866577148438, + "KL/mean": -8.022209167480469, + "KL/rejected_KL_mean": -12.92153549194336, + "KL/std": 10.03524398803711, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.042240217328071594, + "fcm_dpo/delta": -0.024514786899089813, + "fcm_dpo/margin": 9.798646926879883, + "fcm_dpo/q_t": 0.40358948707580566, + "grad_norm": 30.853675842285156, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6088730096817017, + "logits/rejected": -0.5662700533866882, + "logps/chosen": -62.46369171142578, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -85.70881652832031, + "loss": 1.0751, + "margin_dpo/margin_mean": 9.798646926879883, + "margin_dpo/margin_std": 12.186027526855469, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.8463668823242188, + "KL/mean": -8.590458869934082, + "KL/rejected_KL_mean": -13.334552764892578, + "KL/std": 9.031107902526855, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.041834283620119095, + "fcm_dpo/delta": 0.003476101905107498, + "fcm_dpo/margin": 9.48818588256836, + "fcm_dpo/q_t": 0.4069967567920685, + "grad_norm": 29.675251007080078, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6760765314102173, + "logits/rejected": -0.6185558438301086, + "logps/chosen": -69.05220031738281, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -90.54180145263672, + "loss": 1.0714, + "margin_dpo/margin_mean": 9.48818588256836, + "margin_dpo/margin_std": 10.152603149414062, + "step": 68 + }, + { + "KL/chosen_KL_mean": -4.753917694091797, + "KL/mean": -10.989311218261719, + "KL/rejected_KL_mean": -17.224708557128906, + "KL/std": 11.462624549865723, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.034877799451351166, + "fcm_dpo/delta": -0.07517345249652863, + "fcm_dpo/margin": 12.470787048339844, + "fcm_dpo/q_t": 0.39176517724990845, + "grad_norm": 30.214656829833984, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6015123724937439, + "logits/rejected": -0.5792900323867798, + "logps/chosen": -64.57315826416016, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -120.61357116699219, + "loss": 1.0442, + "margin_dpo/margin_mean": 12.470787048339844, + "margin_dpo/margin_std": 12.772706985473633, + "step": 69 + }, + { + "KL/chosen_KL_mean": -7.221284866333008, + "KL/mean": -13.940620422363281, + "KL/rejected_KL_mean": -20.659957885742188, + "KL/std": 14.507774353027344, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.0326962023973465, + "fcm_dpo/delta": -0.06987478584051132, + "fcm_dpo/margin": 13.438671112060547, + "fcm_dpo/q_t": 0.39314454793930054, + "grad_norm": 29.549423217773438, + "learning_rate": 5e-07, + "logits/chosen": -0.6086961627006531, + "logits/rejected": -0.5738969445228577, + "logps/chosen": -69.15192413330078, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -111.72074127197266, + "loss": 1.0564, + "margin_dpo/margin_mean": 13.438671112060547, + "margin_dpo/margin_std": 15.399250984191895, + "step": 70 + }, + { + "KL/chosen_KL_mean": -7.312736511230469, + "KL/mean": -15.442537307739258, + "KL/rejected_KL_mean": -23.572338104248047, + "KL/std": 15.629602432250977, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.026657406240701675, + "fcm_dpo/delta": -0.06790647655725479, + "fcm_dpo/margin": 16.259611129760742, + "fcm_dpo/q_t": 0.3945091962814331, + "grad_norm": 26.020174026489258, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6621614694595337, + "logits/rejected": -0.6253249645233154, + "logps/chosen": -69.06307220458984, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -120.9089584350586, + "loss": 1.0485, + "margin_dpo/margin_mean": 16.259613037109375, + "margin_dpo/margin_std": 17.08768653869629, + "step": 71 + }, + { + "KL/chosen_KL_mean": -8.572813034057617, + "KL/mean": -16.923320770263672, + "KL/rejected_KL_mean": -25.273826599121094, + "KL/std": 16.826129913330078, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.025071250274777412, + "fcm_dpo/delta": -0.03367580845952034, + "fcm_dpo/margin": 16.70101547241211, + "fcm_dpo/q_t": 0.4001430869102478, + "grad_norm": 24.218080520629883, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6549113988876343, + "logits/rejected": -0.617749810218811, + "logps/chosen": -74.626220703125, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -120.5608139038086, + "loss": 1.0698, + "margin_dpo/margin_mean": 16.70101547241211, + "margin_dpo/margin_std": 20.266984939575195, + "step": 72 + }, + { + "KL/chosen_KL_mean": -10.759519577026367, + "KL/mean": -19.278858184814453, + "KL/rejected_KL_mean": -27.798202514648438, + "KL/std": 21.811187744140625, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.02377324365079403, + "fcm_dpo/delta": -0.011964067816734314, + "fcm_dpo/margin": 17.038679122924805, + "fcm_dpo/q_t": 0.40983206033706665, + "grad_norm": 23.957334518432617, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6897900104522705, + "logits/rejected": -0.6500684022903442, + "logps/chosen": -77.01579284667969, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -118.25433349609375, + "loss": 1.1199, + "margin_dpo/margin_mean": 17.038681030273438, + "margin_dpo/margin_std": 27.641098022460938, + "step": 73 + }, + { + "KL/chosen_KL_mean": -11.2384033203125, + "KL/mean": -20.760555267333984, + "KL/rejected_KL_mean": -30.28270721435547, + "KL/std": 24.14501190185547, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.022344253957271576, + "fcm_dpo/delta": -0.0470757819712162, + "fcm_dpo/margin": 19.044301986694336, + "fcm_dpo/q_t": 0.40436846017837524, + "grad_norm": 24.303987503051758, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6677048206329346, + "logits/rejected": -0.6622331142425537, + "logps/chosen": -64.66328430175781, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -126.22964477539062, + "loss": 1.0957, + "margin_dpo/margin_mean": 19.044300079345703, + "margin_dpo/margin_std": 30.3118896484375, + "step": 74 + }, + { + "KL/chosen_KL_mean": -11.227849960327148, + "KL/mean": -24.636085510253906, + "KL/rejected_KL_mean": -38.04431915283203, + "KL/std": 24.73340606689453, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.018182795494794846, + "fcm_dpo/delta": -0.16822758316993713, + "fcm_dpo/margin": 26.81647491455078, + "fcm_dpo/q_t": 0.3744775354862213, + "grad_norm": 22.551315307617188, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.7105200290679932, + "logits/rejected": -0.70478755235672, + "logps/chosen": -63.08951187133789, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -149.29830932617188, + "loss": 1.0253, + "margin_dpo/margin_mean": 26.81647491455078, + "margin_dpo/margin_std": 28.454429626464844, + "step": 75 + }, + { + "KL/chosen_KL_mean": -13.570169448852539, + "KL/mean": -21.888824462890625, + "KL/rejected_KL_mean": -30.207481384277344, + "KL/std": 20.21454620361328, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.019159602001309395, + "fcm_dpo/delta": 0.1141112893819809, + "fcm_dpo/margin": 16.637313842773438, + "fcm_dpo/q_t": 0.43109238147735596, + "grad_norm": 21.629785537719727, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.7143449783325195, + "logits/rejected": -0.6758897304534912, + "logps/chosen": -66.83621215820312, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -108.42411041259766, + "loss": 1.1366, + "margin_dpo/margin_mean": 16.63731575012207, + "margin_dpo/margin_std": 22.22722816467285, + "step": 76 + }, + { + "KL/chosen_KL_mean": -12.503726959228516, + "KL/mean": -26.28228759765625, + "KL/rejected_KL_mean": -40.06085205078125, + "KL/std": 27.661182403564453, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.016155380755662918, + "fcm_dpo/delta": -0.10240481793880463, + "fcm_dpo/margin": 27.55712127685547, + "fcm_dpo/q_t": 0.38546115159988403, + "grad_norm": 21.97159767150879, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6718529462814331, + "logits/rejected": -0.635248601436615, + "logps/chosen": -70.60040283203125, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -133.83447265625, + "loss": 1.0496, + "margin_dpo/margin_mean": 27.55712127685547, + "margin_dpo/margin_std": 31.21552085876465, + "step": 77 + }, + { + "KL/chosen_KL_mean": -12.715435028076172, + "KL/mean": -22.89756202697754, + "KL/rejected_KL_mean": -33.07969284057617, + "KL/std": 22.80880355834961, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.0179641991853714, + "fcm_dpo/delta": 0.04957953840494156, + "fcm_dpo/margin": 20.364255905151367, + "fcm_dpo/q_t": 0.41824695467948914, + "grad_norm": 20.99672508239746, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6376138925552368, + "logits/rejected": -0.6148805618286133, + "logps/chosen": -68.3292236328125, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -118.01405334472656, + "loss": 1.1079, + "margin_dpo/margin_mean": 20.364255905151367, + "margin_dpo/margin_std": 26.547138214111328, + "step": 78 + }, + { + "KL/chosen_KL_mean": -13.37647819519043, + "KL/mean": -24.498859405517578, + "KL/rejected_KL_mean": -35.62124252319336, + "KL/std": 25.48963737487793, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.01794726401567459, + "fcm_dpo/delta": 0.0012769072782248259, + "fcm_dpo/margin": 22.24475860595703, + "fcm_dpo/q_t": 0.41002583503723145, + "grad_norm": 19.79173469543457, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6492550373077393, + "logits/rejected": -0.6207257509231567, + "logps/chosen": -68.82696533203125, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -123.26880645751953, + "loss": 1.0989, + "margin_dpo/margin_mean": 22.24475860595703, + "margin_dpo/margin_std": 31.258174896240234, + "step": 79 + }, + { + "KL/chosen_KL_mean": -17.12893295288086, + "KL/mean": -28.049373626708984, + "KL/rejected_KL_mean": -38.969810485839844, + "KL/std": 28.13406753540039, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.017532825469970703, + "fcm_dpo/delta": 0.020619504153728485, + "fcm_dpo/margin": 21.840877532958984, + "fcm_dpo/q_t": 0.41682252287864685, + "grad_norm": 21.230684280395508, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7177095413208008, + "logits/rejected": -0.6798745393753052, + "logps/chosen": -75.64822387695312, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -126.5173110961914, + "loss": 1.1204, + "margin_dpo/margin_mean": 21.840877532958984, + "margin_dpo/margin_std": 32.85547637939453, + "step": 80 + }, + { + "KL/chosen_KL_mean": -16.53759765625, + "KL/mean": -31.449586868286133, + "KL/rejected_KL_mean": -46.361572265625, + "KL/std": 33.21380615234375, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.015006015077233315, + "fcm_dpo/delta": -0.10901469737291336, + "fcm_dpo/margin": 29.823978424072266, + "fcm_dpo/q_t": 0.3896971344947815, + "grad_norm": 20.577526092529297, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7138460874557495, + "logits/rejected": -0.6892383694648743, + "logps/chosen": -82.98646545410156, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -176.02427673339844, + "loss": 1.0665, + "margin_dpo/margin_mean": 29.823976516723633, + "margin_dpo/margin_std": 39.704833984375, + "step": 81 + }, + { + "KL/chosen_KL_mean": -19.28495216369629, + "KL/mean": -31.137420654296875, + "KL/rejected_KL_mean": -42.98988723754883, + "KL/std": 30.5316104888916, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.016485687345266342, + "fcm_dpo/delta": -0.008920453488826752, + "fcm_dpo/margin": 23.70493507385254, + "fcm_dpo/q_t": 0.4126429855823517, + "grad_norm": 23.881196975708008, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6434165239334106, + "logits/rejected": -0.6364510655403137, + "logps/chosen": -71.517333984375, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -133.73313903808594, + "loss": 1.1577, + "margin_dpo/margin_mean": 23.704936981201172, + "margin_dpo/margin_std": 42.49738311767578, + "step": 82 + }, + { + "KL/chosen_KL_mean": -19.908788681030273, + "KL/mean": -34.38926696777344, + "KL/rejected_KL_mean": -48.869747161865234, + "KL/std": 31.844987869262695, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.014069490134716034, + "fcm_dpo/delta": -0.0153765007853508, + "fcm_dpo/margin": 28.960956573486328, + "fcm_dpo/q_t": 0.4053932726383209, + "grad_norm": 19.797786712646484, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6841074228286743, + "logits/rejected": -0.657940149307251, + "logps/chosen": -75.73617553710938, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -152.5856475830078, + "loss": 1.0873, + "margin_dpo/margin_mean": 28.96095848083496, + "margin_dpo/margin_std": 38.147422790527344, + "step": 83 + }, + { + "KL/chosen_KL_mean": -19.783796310424805, + "KL/mean": -31.47045135498047, + "KL/rejected_KL_mean": -43.157108306884766, + "KL/std": 27.439373016357422, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.015063371509313583, + "fcm_dpo/delta": 0.07021048665046692, + "fcm_dpo/margin": 23.373310089111328, + "fcm_dpo/q_t": 0.4214816689491272, + "grad_norm": 20.031166076660156, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6388710737228394, + "logits/rejected": -0.5998183488845825, + "logps/chosen": -86.9599609375, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -130.45571899414062, + "loss": 1.1181, + "margin_dpo/margin_mean": 23.373310089111328, + "margin_dpo/margin_std": 27.697301864624023, + "step": 84 + }, + { + "KL/chosen_KL_mean": -19.78733253479004, + "KL/mean": -32.103843688964844, + "KL/rejected_KL_mean": -44.42035675048828, + "KL/std": 27.902463912963867, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.016904689371585846, + "fcm_dpo/delta": -0.03945329040288925, + "fcm_dpo/margin": 24.633026123046875, + "fcm_dpo/q_t": 0.4018494188785553, + "grad_norm": 21.946027755737305, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6458739638328552, + "logits/rejected": -0.6141970157623291, + "logps/chosen": -78.19395446777344, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -123.05915832519531, + "loss": 1.099, + "margin_dpo/margin_mean": 24.633026123046875, + "margin_dpo/margin_std": 33.796836853027344, + "step": 85 + }, + { + "KL/chosen_KL_mean": -26.077333450317383, + "KL/mean": -39.15126037597656, + "KL/rejected_KL_mean": -52.225189208984375, + "KL/std": 39.593040466308594, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.015605635941028595, + "fcm_dpo/delta": -0.01844988390803337, + "fcm_dpo/margin": 26.14786148071289, + "fcm_dpo/q_t": 0.4177693724632263, + "grad_norm": 25.58841896057129, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6518966555595398, + "logits/rejected": -0.6431847810745239, + "logps/chosen": -82.21479797363281, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -140.34683227539062, + "loss": 1.1793, + "margin_dpo/margin_mean": 26.147863388061523, + "margin_dpo/margin_std": 55.436424255371094, + "step": 86 + }, + { + "KL/chosen_KL_mean": -25.545059204101562, + "KL/mean": -41.57789611816406, + "KL/rejected_KL_mean": -57.61074447631836, + "KL/std": 39.62342071533203, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.013056030496954918, + "fcm_dpo/delta": -0.044095173478126526, + "fcm_dpo/margin": 32.0656852722168, + "fcm_dpo/q_t": 0.40036922693252563, + "grad_norm": 19.81938362121582, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.6956198215484619, + "logits/rejected": -0.6780788898468018, + "logps/chosen": -81.18115234375, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -153.0783233642578, + "loss": 1.1109, + "margin_dpo/margin_mean": 32.0656852722168, + "margin_dpo/margin_std": 48.967384338378906, + "step": 87 + }, + { + "KL/chosen_KL_mean": -32.21333694458008, + "KL/mean": -44.31005096435547, + "KL/rejected_KL_mean": -56.40675735473633, + "KL/std": 42.025611877441406, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.014491596259176731, + "fcm_dpo/delta": 0.07550811022520065, + "fcm_dpo/margin": 24.193424224853516, + "fcm_dpo/q_t": 0.4285997748374939, + "grad_norm": 21.64423179626465, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6556172370910645, + "logits/rejected": -0.6542757749557495, + "logps/chosen": -105.88449096679688, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -163.11526489257812, + "loss": 1.1734, + "margin_dpo/margin_mean": 24.193424224853516, + "margin_dpo/margin_std": 45.42322540283203, + "step": 88 + }, + { + "KL/chosen_KL_mean": -20.79347801208496, + "KL/mean": -34.439876556396484, + "KL/rejected_KL_mean": -48.08627700805664, + "KL/std": 35.92295837402344, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.01547418162226677, + "fcm_dpo/delta": -0.04180596396327019, + "fcm_dpo/margin": 27.29279136657715, + "fcm_dpo/q_t": 0.4021642208099365, + "grad_norm": 21.672611236572266, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7387075424194336, + "logits/rejected": -0.7136551141738892, + "logps/chosen": -81.41839599609375, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -130.16983032226562, + "loss": 1.1057, + "margin_dpo/margin_mean": 27.29279327392578, + "margin_dpo/margin_std": 42.178958892822266, + "step": 89 + }, + { + "KL/chosen_KL_mean": -25.24016761779785, + "KL/mean": -44.94635772705078, + "KL/rejected_KL_mean": -64.65254211425781, + "KL/std": 48.8939208984375, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.012419058009982109, + "fcm_dpo/delta": -0.1753031611442566, + "fcm_dpo/margin": 39.412376403808594, + "fcm_dpo/q_t": 0.38558393716812134, + "grad_norm": 20.073680877685547, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6283840537071228, + "logits/rejected": -0.6491606831550598, + "logps/chosen": -78.52548217773438, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -176.197265625, + "loss": 1.1068, + "margin_dpo/margin_mean": 39.41238021850586, + "margin_dpo/margin_std": 61.327945709228516, + "step": 90 + }, + { + "KL/chosen_KL_mean": -27.13432502746582, + "KL/mean": -41.85686111450195, + "KL/rejected_KL_mean": -56.57939910888672, + "KL/std": 37.675636291503906, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.011744784191250801, + "fcm_dpo/delta": 0.07527971267700195, + "fcm_dpo/margin": 29.445079803466797, + "fcm_dpo/q_t": 0.42634034156799316, + "grad_norm": 17.86397361755371, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6645762920379639, + "logits/rejected": -0.635959267616272, + "logps/chosen": -88.93728637695312, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -144.45335388183594, + "loss": 1.151, + "margin_dpo/margin_mean": 29.445079803466797, + "margin_dpo/margin_std": 50.49188995361328, + "step": 91 + }, + { + "KL/chosen_KL_mean": -23.93476676940918, + "KL/mean": -39.31444549560547, + "KL/rejected_KL_mean": -54.69412612915039, + "KL/std": 39.209224700927734, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.012650266289710999, + "fcm_dpo/delta": 0.008795984089374542, + "fcm_dpo/margin": 30.75935935974121, + "fcm_dpo/q_t": 0.41261714696884155, + "grad_norm": 18.466337203979492, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.6946055889129639, + "logits/rejected": -0.6752103567123413, + "logps/chosen": -75.57553100585938, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -132.5753173828125, + "loss": 1.1221, + "margin_dpo/margin_mean": 30.759361267089844, + "margin_dpo/margin_std": 47.41279602050781, + "step": 92 + }, + { + "KL/chosen_KL_mean": -27.03689193725586, + "KL/mean": -42.25169372558594, + "KL/rejected_KL_mean": -57.466495513916016, + "KL/std": 36.04038619995117, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.012704822234809399, + "fcm_dpo/delta": 0.01873387210071087, + "fcm_dpo/margin": 30.429601669311523, + "fcm_dpo/q_t": 0.4136204719543457, + "grad_norm": 19.59035873413086, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7035015821456909, + "logits/rejected": -0.6627704501152039, + "logps/chosen": -79.56613159179688, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -134.6272430419922, + "loss": 1.103, + "margin_dpo/margin_mean": 30.42959976196289, + "margin_dpo/margin_std": 42.63709259033203, + "step": 93 + }, + { + "KL/chosen_KL_mean": -28.926584243774414, + "KL/mean": -47.09135818481445, + "KL/rejected_KL_mean": -65.25613403320312, + "KL/std": 42.653038024902344, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.011295975185930729, + "fcm_dpo/delta": -0.028756409883499146, + "fcm_dpo/margin": 36.32954788208008, + "fcm_dpo/q_t": 0.40284472703933716, + "grad_norm": 17.930683135986328, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.626760721206665, + "logits/rejected": -0.618344783782959, + "logps/chosen": -90.14920043945312, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -164.85516357421875, + "loss": 1.0816, + "margin_dpo/margin_mean": 36.32954788208008, + "margin_dpo/margin_std": 45.59572219848633, + "step": 94 + }, + { + "KL/chosen_KL_mean": -28.432294845581055, + "KL/mean": -43.03887176513672, + "KL/rejected_KL_mean": -57.64545440673828, + "KL/std": 43.11521530151367, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.012412654235959053, + "fcm_dpo/delta": 0.05812495946884155, + "fcm_dpo/margin": 29.213159561157227, + "fcm_dpo/q_t": 0.42024633288383484, + "grad_norm": 19.021087646484375, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.694536030292511, + "logits/rejected": -0.673601508140564, + "logps/chosen": -80.95594024658203, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -133.5258026123047, + "loss": 1.133, + "margin_dpo/margin_mean": 29.213159561157227, + "margin_dpo/margin_std": 45.21534729003906, + "step": 95 + }, + { + "KL/chosen_KL_mean": -28.583940505981445, + "KL/mean": -48.43400573730469, + "KL/rejected_KL_mean": -68.28407287597656, + "KL/std": 46.205230712890625, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.010704778134822845, + "fcm_dpo/delta": -0.0798785462975502, + "fcm_dpo/margin": 39.70012664794922, + "fcm_dpo/q_t": 0.3951551914215088, + "grad_norm": 17.751604080200195, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.6716827154159546, + "logits/rejected": -0.6496946811676025, + "logps/chosen": -90.74091339111328, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -164.88009643554688, + "loss": 1.0813, + "margin_dpo/margin_mean": 39.70012664794922, + "margin_dpo/margin_std": 52.09516906738281, + "step": 96 + }, + { + "KL/chosen_KL_mean": -29.381603240966797, + "KL/mean": -44.338138580322266, + "KL/rejected_KL_mean": -59.294677734375, + "KL/std": 39.46461486816406, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.010876108892261982, + "fcm_dpo/delta": -0.03550150245428085, + "fcm_dpo/margin": 29.913070678710938, + "fcm_dpo/q_t": 0.4207790791988373, + "grad_norm": 18.363679885864258, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.5955668091773987, + "logits/rejected": -0.5594468116760254, + "logps/chosen": -84.02796936035156, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -136.25942993164062, + "loss": 1.1386, + "margin_dpo/margin_mean": 29.913070678710938, + "margin_dpo/margin_std": 39.46156311035156, + "step": 97 + }, + { + "KL/chosen_KL_mean": -37.65998458862305, + "KL/mean": -54.85460662841797, + "KL/rejected_KL_mean": -72.04922485351562, + "KL/std": 45.98478698730469, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.0109539944678545, + "fcm_dpo/delta": 0.03750079125165939, + "fcm_dpo/margin": 34.38923645019531, + "fcm_dpo/q_t": 0.4147164225578308, + "grad_norm": 20.02430534362793, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.652593731880188, + "logits/rejected": -0.6433310508728027, + "logps/chosen": -102.91860961914062, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -168.57672119140625, + "loss": 1.1337, + "margin_dpo/margin_mean": 34.38923645019531, + "margin_dpo/margin_std": 55.96516799926758, + "step": 98 + }, + { + "KL/chosen_KL_mean": -28.500194549560547, + "KL/mean": -50.64104461669922, + "KL/rejected_KL_mean": -72.78189849853516, + "KL/std": 48.286502838134766, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.00977346021682024, + "fcm_dpo/delta": -0.06876382231712341, + "fcm_dpo/margin": 44.28170394897461, + "fcm_dpo/q_t": 0.3981579542160034, + "grad_norm": 16.403064727783203, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6286486387252808, + "logits/rejected": -0.6337354183197021, + "logps/chosen": -74.13867950439453, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -159.21983337402344, + "loss": 1.0777, + "margin_dpo/margin_mean": 44.28170394897461, + "margin_dpo/margin_std": 60.85698699951172, + "step": 99 + }, + { + "KL/chosen_KL_mean": -32.810997009277344, + "KL/mean": -44.14629364013672, + "KL/rejected_KL_mean": -55.4815788269043, + "KL/std": 39.670440673828125, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.010382324457168579, + "fcm_dpo/delta": 0.05089900270104408, + "fcm_dpo/margin": 22.670581817626953, + "fcm_dpo/q_t": 0.4473540782928467, + "grad_norm": 18.2591495513916, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6274121999740601, + "logits/rejected": -0.5915518999099731, + "logps/chosen": -90.40498352050781, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -129.5417938232422, + "loss": 1.23, + "margin_dpo/margin_mean": 22.670583724975586, + "margin_dpo/margin_std": 50.92362976074219, + "step": 100 + }, + { + "KL/chosen_KL_mean": -40.26070022583008, + "KL/mean": -54.877384185791016, + "KL/rejected_KL_mean": -69.49407958984375, + "KL/std": 49.381065368652344, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.010775712318718433, + "fcm_dpo/delta": -0.0062071713618934155, + "fcm_dpo/margin": 29.23336410522461, + "fcm_dpo/q_t": 0.43012934923171997, + "grad_norm": 19.166645050048828, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6265345811843872, + "logits/rejected": -0.6024205684661865, + "logps/chosen": -101.90955352783203, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -152.6837615966797, + "loss": 1.1904, + "margin_dpo/margin_mean": 29.23336410522461, + "margin_dpo/margin_std": 58.819854736328125, + "step": 101 + }, + { + "KL/chosen_KL_mean": -42.29863739013672, + "KL/mean": -52.66799545288086, + "KL/rejected_KL_mean": -63.037353515625, + "KL/std": 45.68564224243164, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.011057091876864433, + "fcm_dpo/delta": 0.02545342966914177, + "fcm_dpo/margin": 20.73870849609375, + "fcm_dpo/q_t": 0.4484948515892029, + "grad_norm": 22.39993667602539, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.6203207969665527, + "logits/rejected": -0.580852210521698, + "logps/chosen": -106.37751770019531, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -131.22442626953125, + "loss": 1.2607, + "margin_dpo/margin_mean": 20.73870849609375, + "margin_dpo/margin_std": 55.18003463745117, + "step": 102 + }, + { + "KL/chosen_KL_mean": -35.84786605834961, + "KL/mean": -58.045082092285156, + "KL/rejected_KL_mean": -80.24229431152344, + "KL/std": 51.89337921142578, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.009846476837992668, + "fcm_dpo/delta": -0.07356812059879303, + "fcm_dpo/margin": 44.39442443847656, + "fcm_dpo/q_t": 0.3977162837982178, + "grad_norm": 18.495681762695312, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6326062679290771, + "logits/rejected": -0.603610634803772, + "logps/chosen": -97.14714050292969, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -173.81500244140625, + "loss": 1.0847, + "margin_dpo/margin_mean": 44.39442443847656, + "margin_dpo/margin_std": 64.85426330566406, + "step": 103 + }, + { + "KL/chosen_KL_mean": -36.992637634277344, + "KL/mean": -58.22779846191406, + "KL/rejected_KL_mean": -79.46296691894531, + "KL/std": 55.633026123046875, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.00963391549885273, + "fcm_dpo/delta": -0.015489146113395691, + "fcm_dpo/margin": 42.47032928466797, + "fcm_dpo/q_t": 0.410830020904541, + "grad_norm": 17.655324935913086, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6385079622268677, + "logits/rejected": -0.6165839433670044, + "logps/chosen": -91.36540985107422, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -169.02767944335938, + "loss": 1.1177, + "margin_dpo/margin_mean": 42.47032928466797, + "margin_dpo/margin_std": 69.24513244628906, + "step": 104 + }, + { + "KL/chosen_KL_mean": -35.22704315185547, + "KL/mean": -65.53314208984375, + "KL/rejected_KL_mean": -95.83924865722656, + "KL/std": 52.481048583984375, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.007748795207589865, + "fcm_dpo/delta": -0.13560038805007935, + "fcm_dpo/margin": 60.61220932006836, + "fcm_dpo/q_t": 0.37679198384284973, + "grad_norm": 16.846649169921875, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.5944653749465942, + "logits/rejected": -0.5763468742370605, + "logps/chosen": -89.86598205566406, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -193.81277465820312, + "loss": 1.0206, + "margin_dpo/margin_mean": 60.612213134765625, + "margin_dpo/margin_std": 59.409732818603516, + "step": 105 + }, + { + "KL/chosen_KL_mean": -40.11956024169922, + "KL/mean": -58.95640563964844, + "KL/rejected_KL_mean": -77.79325866699219, + "KL/std": 47.83055114746094, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.0085395947098732, + "fcm_dpo/delta": 0.11511291563510895, + "fcm_dpo/margin": 37.6736946105957, + "fcm_dpo/q_t": 0.43202945590019226, + "grad_norm": 16.822023391723633, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.5823087096214294, + "logits/rejected": -0.56536865234375, + "logps/chosen": -94.95245361328125, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -163.01788330078125, + "loss": 1.1419, + "margin_dpo/margin_mean": 37.67369079589844, + "margin_dpo/margin_std": 51.76994323730469, + "step": 106 + }, + { + "KL/chosen_KL_mean": -47.187808990478516, + "KL/mean": -68.33793640136719, + "KL/rejected_KL_mean": -89.4880599975586, + "KL/std": 58.32842254638672, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.0090809166431427, + "fcm_dpo/delta": 0.023154791444540024, + "fcm_dpo/margin": 42.300254821777344, + "fcm_dpo/q_t": 0.41329509019851685, + "grad_norm": 18.076017379760742, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6128166913986206, + "logits/rejected": -0.5746489763259888, + "logps/chosen": -116.89562225341797, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -184.22756958007812, + "loss": 1.1189, + "margin_dpo/margin_mean": 42.300254821777344, + "margin_dpo/margin_std": 62.243568420410156, + "step": 107 + }, + { + "KL/chosen_KL_mean": -40.908172607421875, + "KL/mean": -67.95404052734375, + "KL/rejected_KL_mean": -94.99990844726562, + "KL/std": 65.11196899414062, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.008320159278810024, + "fcm_dpo/delta": -0.09254781901836395, + "fcm_dpo/margin": 54.09173583984375, + "fcm_dpo/q_t": 0.3973488211631775, + "grad_norm": 16.829160690307617, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.5930562019348145, + "logits/rejected": -0.5809206962585449, + "logps/chosen": -96.91806030273438, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -190.79592895507812, + "loss": 1.0985, + "margin_dpo/margin_mean": 54.09173583984375, + "margin_dpo/margin_std": 84.54571533203125, + "step": 108 + }, + { + "KL/chosen_KL_mean": -39.185829162597656, + "KL/mean": -66.94438171386719, + "KL/rejected_KL_mean": -94.70292663574219, + "KL/std": 61.541465759277344, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.007376614026725292, + "fcm_dpo/delta": -0.021881788969039917, + "fcm_dpo/margin": 55.51709747314453, + "fcm_dpo/q_t": 0.4035341143608093, + "grad_norm": 17.845430374145508, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5359349250793457, + "logits/rejected": -0.5162516832351685, + "logps/chosen": -102.07131958007812, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -193.388671875, + "loss": 1.0947, + "margin_dpo/margin_mean": 55.51709747314453, + "margin_dpo/margin_std": 77.26396179199219, + "step": 109 + }, + { + "KL/chosen_KL_mean": -41.10560989379883, + "KL/mean": -66.01383972167969, + "KL/rejected_KL_mean": -90.92208099365234, + "KL/std": 61.80369567871094, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.006370611023157835, + "fcm_dpo/delta": -0.08743350207805634, + "fcm_dpo/margin": 49.81646728515625, + "fcm_dpo/q_t": 0.41799625754356384, + "grad_norm": 12.691235542297363, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.5346049666404724, + "logits/rejected": -0.5050790309906006, + "logps/chosen": -99.85929870605469, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -170.67208862304688, + "loss": 1.1549, + "margin_dpo/margin_mean": 49.81646728515625, + "margin_dpo/margin_std": 75.78208923339844, + "step": 110 + }, + { + "KL/chosen_KL_mean": -45.89485168457031, + "KL/mean": -75.09423828125, + "KL/rejected_KL_mean": -104.29363250732422, + "KL/std": 70.21835327148438, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.006470158696174622, + "fcm_dpo/delta": 0.03387834131717682, + "fcm_dpo/margin": 58.398780822753906, + "fcm_dpo/q_t": 0.41569167375564575, + "grad_norm": 14.799457550048828, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.5766105055809021, + "logits/rejected": -0.5566138029098511, + "logps/chosen": -114.51895904541016, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -202.72250366210938, + "loss": 1.1264, + "margin_dpo/margin_mean": 58.39877700805664, + "margin_dpo/margin_std": 89.97933959960938, + "step": 111 + }, + { + "KL/chosen_KL_mean": -42.42522048950195, + "KL/mean": -60.012123107910156, + "KL/rejected_KL_mean": -77.59901428222656, + "KL/std": 46.559471130371094, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.00751154450699687, + "fcm_dpo/delta": 0.04849451780319214, + "fcm_dpo/margin": 35.173797607421875, + "fcm_dpo/q_t": 0.4408394694328308, + "grad_norm": 15.346757888793945, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.53926682472229, + "logits/rejected": -0.5091655254364014, + "logps/chosen": -92.67486572265625, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -142.37344360351562, + "loss": 1.1874, + "margin_dpo/margin_mean": 35.173797607421875, + "margin_dpo/margin_std": 58.55726623535156, + "step": 112 + }, + { + "KL/chosen_KL_mean": -55.43156433105469, + "KL/mean": -73.22611999511719, + "KL/rejected_KL_mean": -91.02066802978516, + "KL/std": 52.921966552734375, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.009013749659061432, + "fcm_dpo/delta": 0.11262636631727219, + "fcm_dpo/margin": 35.58909606933594, + "fcm_dpo/q_t": 0.4331721365451813, + "grad_norm": 19.263153076171875, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.5204916596412659, + "logits/rejected": -0.483734130859375, + "logps/chosen": -122.14451599121094, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -168.9893798828125, + "loss": 1.1549, + "margin_dpo/margin_mean": 35.58909606933594, + "margin_dpo/margin_std": 58.14237976074219, + "step": 113 + }, + { + "KL/chosen_KL_mean": -49.490875244140625, + "KL/mean": -79.34724426269531, + "KL/rejected_KL_mean": -109.20361328125, + "KL/std": 74.02787780761719, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.007360486313700676, + "fcm_dpo/delta": -0.10784411430358887, + "fcm_dpo/margin": 59.712745666503906, + "fcm_dpo/q_t": 0.3967573046684265, + "grad_norm": 18.57329750061035, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.5290780067443848, + "logits/rejected": -0.5046583414077759, + "logps/chosen": -107.27595520019531, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -196.3132781982422, + "loss": 1.0889, + "margin_dpo/margin_mean": 59.712745666503906, + "margin_dpo/margin_std": 90.68304443359375, + "step": 114 + }, + { + "KL/chosen_KL_mean": -62.15449523925781, + "KL/mean": -82.36180114746094, + "KL/rejected_KL_mean": -102.5691146850586, + "KL/std": 71.60066223144531, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.008717365562915802, + "fcm_dpo/delta": 0.06397520750761032, + "fcm_dpo/margin": 40.414615631103516, + "fcm_dpo/q_t": 0.4292001724243164, + "grad_norm": 26.535457611083984, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.5466749668121338, + "logits/rejected": -0.5373319387435913, + "logps/chosen": -127.73713684082031, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -201.1346435546875, + "loss": 1.2197, + "margin_dpo/margin_mean": 40.41461944580078, + "margin_dpo/margin_std": 96.31269073486328, + "step": 115 + }, + { + "KL/chosen_KL_mean": -45.30133819580078, + "KL/mean": -68.48527526855469, + "KL/rejected_KL_mean": -91.66920471191406, + "KL/std": 58.61573028564453, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.008865797892212868, + "fcm_dpo/delta": -0.027695897966623306, + "fcm_dpo/margin": 46.367855072021484, + "fcm_dpo/q_t": 0.4044142961502075, + "grad_norm": 19.043041229248047, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.5231291055679321, + "logits/rejected": -0.5219400525093079, + "logps/chosen": -96.70165252685547, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -172.19105529785156, + "loss": 1.1011, + "margin_dpo/margin_mean": 46.367855072021484, + "margin_dpo/margin_std": 64.93318939208984, + "step": 116 + }, + { + "KL/chosen_KL_mean": -58.65241241455078, + "KL/mean": -76.01322937011719, + "KL/rejected_KL_mean": -93.37403869628906, + "KL/std": 61.22138977050781, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.009560597129166126, + "fcm_dpo/delta": 0.09886346757411957, + "fcm_dpo/margin": 34.72163009643555, + "fcm_dpo/q_t": 0.4316639304161072, + "grad_norm": 31.035114288330078, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.49749547243118286, + "logits/rejected": -0.46514734625816345, + "logps/chosen": -127.95082092285156, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -159.95803833007812, + "loss": 1.1874, + "margin_dpo/margin_mean": 34.72163391113281, + "margin_dpo/margin_std": 71.40594482421875, + "step": 117 + }, + { + "KL/chosen_KL_mean": -44.07014465332031, + "KL/mean": -66.22931671142578, + "KL/rejected_KL_mean": -88.38848114013672, + "KL/std": 55.144588470458984, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.009519684128463268, + "fcm_dpo/delta": -0.037742018699645996, + "fcm_dpo/margin": 44.318336486816406, + "fcm_dpo/q_t": 0.40347346663475037, + "grad_norm": 20.083301544189453, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.563313364982605, + "logits/rejected": -0.5480994582176208, + "logps/chosen": -99.71112060546875, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -164.0575408935547, + "loss": 1.0928, + "margin_dpo/margin_mean": 44.318336486816406, + "margin_dpo/margin_std": 63.96471405029297, + "step": 118 + }, + { + "KL/chosen_KL_mean": -60.53836441040039, + "KL/mean": -82.12613677978516, + "KL/rejected_KL_mean": -103.71391296386719, + "KL/std": 61.16850280761719, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.00799720548093319, + "fcm_dpo/delta": -0.07085751742124557, + "fcm_dpo/margin": 43.175559997558594, + "fcm_dpo/q_t": 0.41732871532440186, + "grad_norm": 19.45294952392578, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.553020715713501, + "logits/rejected": -0.545320987701416, + "logps/chosen": -134.04855346679688, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -206.6912078857422, + "loss": 1.1596, + "margin_dpo/margin_mean": 43.175559997558594, + "margin_dpo/margin_std": 76.69107055664062, + "step": 119 + }, + { + "KL/chosen_KL_mean": -60.02840805053711, + "KL/mean": -90.28742980957031, + "KL/rejected_KL_mean": -120.54644775390625, + "KL/std": 74.63396453857422, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.007171543315052986, + "fcm_dpo/delta": -0.06302839517593384, + "fcm_dpo/margin": 60.51802444458008, + "fcm_dpo/q_t": 0.3962337374687195, + "grad_norm": 22.441465377807617, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.5601464509963989, + "logits/rejected": -0.5344448089599609, + "logps/chosen": -136.80923461914062, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -228.5701904296875, + "loss": 1.0734, + "margin_dpo/margin_mean": 60.51802062988281, + "margin_dpo/margin_std": 79.95449829101562, + "step": 120 + }, + { + "KL/chosen_KL_mean": -56.952545166015625, + "KL/mean": -91.06138610839844, + "KL/rejected_KL_mean": -125.17024230957031, + "KL/std": 72.70287322998047, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.0062125981785357, + "fcm_dpo/delta": -0.05240712687373161, + "fcm_dpo/margin": 68.21770477294922, + "fcm_dpo/q_t": 0.39837193489074707, + "grad_norm": 17.03141212463379, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5124868154525757, + "logits/rejected": -0.511413037776947, + "logps/chosen": -118.74243927001953, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -235.16481018066406, + "loss": 1.0811, + "margin_dpo/margin_mean": 68.21769714355469, + "margin_dpo/margin_std": 90.72044372558594, + "step": 121 + }, + { + "KL/chosen_KL_mean": -51.569435119628906, + "KL/mean": -93.51173400878906, + "KL/rejected_KL_mean": -135.4540252685547, + "KL/std": 81.56539916992188, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.005438469350337982, + "fcm_dpo/delta": -0.10527084767818451, + "fcm_dpo/margin": 83.88459777832031, + "fcm_dpo/q_t": 0.3847019076347351, + "grad_norm": 17.03737449645996, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.4612678587436676, + "logits/rejected": -0.4794490337371826, + "logps/chosen": -98.47164916992188, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -242.168212890625, + "loss": 1.0341, + "margin_dpo/margin_mean": 83.88459777832031, + "margin_dpo/margin_std": 86.80702209472656, + "step": 122 + }, + { + "KL/chosen_KL_mean": -59.32091522216797, + "KL/mean": -86.37959289550781, + "KL/rejected_KL_mean": -113.43827819824219, + "KL/std": 70.35043334960938, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.005368525628000498, + "fcm_dpo/delta": 0.021436911076307297, + "fcm_dpo/margin": 54.11735534667969, + "fcm_dpo/q_t": 0.4345107674598694, + "grad_norm": 15.537219047546387, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.4674427807331085, + "logits/rejected": -0.4554457664489746, + "logps/chosen": -120.65955352783203, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -201.2136688232422, + "loss": 1.1808, + "margin_dpo/margin_mean": 54.11735534667969, + "margin_dpo/margin_std": 95.79205322265625, + "step": 123 + }, + { + "KL/chosen_KL_mean": -70.40459442138672, + "KL/mean": -111.80335998535156, + "KL/rejected_KL_mean": -153.20213317871094, + "KL/std": 99.7587890625, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.00502355583012104, + "fcm_dpo/delta": -0.029820293188095093, + "fcm_dpo/margin": 82.79753112792969, + "fcm_dpo/q_t": 0.40601056814193726, + "grad_norm": 14.011857986450195, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.4481953978538513, + "logits/rejected": -0.4538896083831787, + "logps/chosen": -141.85293579101562, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -270.78271484375, + "loss": 1.1052, + "margin_dpo/margin_mean": 82.79753875732422, + "margin_dpo/margin_std": 125.91355895996094, + "step": 124 + }, + { + "KL/chosen_KL_mean": -57.097442626953125, + "KL/mean": -92.41481018066406, + "KL/rejected_KL_mean": -127.73218536376953, + "KL/std": 83.84275817871094, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.005386849865317345, + "fcm_dpo/delta": 0.029525037854909897, + "fcm_dpo/margin": 70.63475036621094, + "fcm_dpo/q_t": 0.4158494174480438, + "grad_norm": 14.315393447875977, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.4188230633735657, + "logits/rejected": -0.42580023407936096, + "logps/chosen": -107.23438262939453, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -211.72079467773438, + "loss": 1.1187, + "margin_dpo/margin_mean": 70.63475036621094, + "margin_dpo/margin_std": 106.31303405761719, + "step": 125 + }, + { + "KL/chosen_KL_mean": -62.320526123046875, + "KL/mean": -96.66703796386719, + "KL/rejected_KL_mean": -131.0135498046875, + "KL/std": 78.08503723144531, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.005546340718865395, + "fcm_dpo/delta": 0.030749067664146423, + "fcm_dpo/margin": 68.69302368164062, + "fcm_dpo/q_t": 0.4163479208946228, + "grad_norm": 15.782130241394043, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.42107057571411133, + "logits/rejected": -0.42162832617759705, + "logps/chosen": -117.98759460449219, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -229.143310546875, + "loss": 1.1145, + "margin_dpo/margin_mean": 68.69302368164062, + "margin_dpo/margin_std": 100.53376007080078, + "step": 126 + }, + { + "KL/chosen_KL_mean": -61.44911193847656, + "KL/mean": -84.82119750976562, + "KL/rejected_KL_mean": -108.19326782226562, + "KL/std": 68.35600280761719, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.006859184708446264, + "fcm_dpo/delta": 0.11228857934474945, + "fcm_dpo/margin": 46.74416732788086, + "fcm_dpo/q_t": 0.43170320987701416, + "grad_norm": 19.79861831665039, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.45557230710983276, + "logits/rejected": -0.44303327798843384, + "logps/chosen": -118.00379180908203, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -184.98904418945312, + "loss": 1.1732, + "margin_dpo/margin_mean": 46.744171142578125, + "margin_dpo/margin_std": 86.41580200195312, + "step": 127 + }, + { + "KL/chosen_KL_mean": -64.04533386230469, + "KL/mean": -89.68724060058594, + "KL/rejected_KL_mean": -115.32914733886719, + "KL/std": 80.13230895996094, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.007653497625142336, + "fcm_dpo/delta": 0.0008252784609794617, + "fcm_dpo/margin": 51.28382110595703, + "fcm_dpo/q_t": 0.4136595129966736, + "grad_norm": 32.553951263427734, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.439397931098938, + "logits/rejected": -0.42928874492645264, + "logps/chosen": -122.16629028320312, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -191.76809692382812, + "loss": 1.1786, + "margin_dpo/margin_mean": 51.283817291259766, + "margin_dpo/margin_std": 102.103515625, + "step": 128 + }, + { + "KL/chosen_KL_mean": -78.18240356445312, + "KL/mean": -104.9556884765625, + "KL/rejected_KL_mean": -131.72897338867188, + "KL/std": 91.75530242919922, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.0073263440281152725, + "fcm_dpo/delta": 0.01247208472341299, + "fcm_dpo/margin": 53.54655075073242, + "fcm_dpo/q_t": 0.4221315085887909, + "grad_norm": 23.202054977416992, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.4406889081001282, + "logits/rejected": -0.4349208474159241, + "logps/chosen": -145.09878540039062, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -228.37118530273438, + "loss": 1.1973, + "margin_dpo/margin_mean": 53.54655456542969, + "margin_dpo/margin_std": 122.08485412597656, + "step": 129 + }, + { + "KL/chosen_KL_mean": -56.172401428222656, + "KL/mean": -96.15986633300781, + "KL/rejected_KL_mean": -136.1473388671875, + "KL/std": 85.45478820800781, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.006229862105101347, + "fcm_dpo/delta": -0.19545677304267883, + "fcm_dpo/margin": 79.97492980957031, + "fcm_dpo/q_t": 0.37694329023361206, + "grad_norm": 17.230426788330078, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.42614853382110596, + "logits/rejected": -0.4212262034416199, + "logps/chosen": -100.83924865722656, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -218.92898559570312, + "loss": 1.0744, + "margin_dpo/margin_mean": 79.97492980957031, + "margin_dpo/margin_std": 107.18331146240234, + "step": 130 + }, + { + "KL/chosen_KL_mean": -52.116981506347656, + "KL/mean": -93.28705596923828, + "KL/rejected_KL_mean": -134.45712280273438, + "KL/std": 90.56842041015625, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.004888694733381271, + "fcm_dpo/delta": -0.004482886288315058, + "fcm_dpo/margin": 82.34013366699219, + "fcm_dpo/q_t": 0.40506911277770996, + "grad_norm": 26.67673683166504, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.42915642261505127, + "logits/rejected": -0.4353182315826416, + "logps/chosen": -97.04157257080078, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -222.90113830566406, + "loss": 1.0791, + "margin_dpo/margin_mean": 82.34013366699219, + "margin_dpo/margin_std": 98.78244018554688, + "step": 131 + }, + { + "KL/chosen_KL_mean": -67.49314880371094, + "KL/mean": -99.89106750488281, + "KL/rejected_KL_mean": -132.28900146484375, + "KL/std": 87.30780029296875, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.005424443632364273, + "fcm_dpo/delta": 0.07522791624069214, + "fcm_dpo/margin": 64.79584503173828, + "fcm_dpo/q_t": 0.42780178785324097, + "grad_norm": 16.56679916381836, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.46568989753723145, + "logits/rejected": -0.4727591872215271, + "logps/chosen": -126.49423217773438, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -220.18115234375, + "loss": 1.1552, + "margin_dpo/margin_mean": 64.79584503173828, + "margin_dpo/margin_std": 114.17736053466797, + "step": 132 + }, + { + "KL/chosen_KL_mean": -78.84221649169922, + "KL/mean": -106.69076538085938, + "KL/rejected_KL_mean": -134.53932189941406, + "KL/std": 77.18131256103516, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.006308811716735363, + "fcm_dpo/delta": 0.0753559023141861, + "fcm_dpo/margin": 55.69710159301758, + "fcm_dpo/q_t": 0.4273163676261902, + "grad_norm": 21.23187255859375, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.46139687299728394, + "logits/rejected": -0.4528288245201111, + "logps/chosen": -145.44671630859375, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -230.8728790283203, + "loss": 1.157, + "margin_dpo/margin_mean": 55.69709777832031, + "margin_dpo/margin_std": 99.82786560058594, + "step": 133 + }, + { + "KL/chosen_KL_mean": -59.535091400146484, + "KL/mean": -90.11434173583984, + "KL/rejected_KL_mean": -120.69359588623047, + "KL/std": 71.90062713623047, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.006723651196807623, + "fcm_dpo/delta": -0.022656653076410294, + "fcm_dpo/margin": 61.15849304199219, + "fcm_dpo/q_t": 0.40606844425201416, + "grad_norm": 17.838544845581055, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.4176589846611023, + "logits/rejected": -0.4087156057357788, + "logps/chosen": -111.6043472290039, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -208.34811401367188, + "loss": 1.0923, + "margin_dpo/margin_mean": 61.15850067138672, + "margin_dpo/margin_std": 85.12577056884766, + "step": 134 + }, + { + "KL/chosen_KL_mean": -64.89012145996094, + "KL/mean": -110.44607543945312, + "KL/rejected_KL_mean": -156.00204467773438, + "KL/std": 98.89341735839844, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.005103720352053642, + "fcm_dpo/delta": -0.13181009888648987, + "fcm_dpo/margin": 91.11190795898438, + "fcm_dpo/q_t": 0.3853245973587036, + "grad_norm": 24.731225967407227, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.41161245107650757, + "logits/rejected": -0.4464326500892639, + "logps/chosen": -115.24398803710938, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -271.9817810058594, + "loss": 1.061, + "margin_dpo/margin_mean": 91.11190795898438, + "margin_dpo/margin_std": 121.44206237792969, + "step": 135 + }, + { + "KL/chosen_KL_mean": -77.75067138671875, + "KL/mean": -107.7598876953125, + "KL/rejected_KL_mean": -137.76910400390625, + "KL/std": 92.06062316894531, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.005009594839066267, + "fcm_dpo/delta": 0.013183820061385632, + "fcm_dpo/margin": 60.01844024658203, + "fcm_dpo/q_t": 0.4368482232093811, + "grad_norm": 20.24871063232422, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.41852521896362305, + "logits/rejected": -0.41355031728744507, + "logps/chosen": -142.82318115234375, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -234.09033203125, + "loss": 1.1918, + "margin_dpo/margin_mean": 60.01844024658203, + "margin_dpo/margin_std": 120.7431640625, + "step": 136 + }, + { + "KL/chosen_KL_mean": -70.75765991210938, + "KL/mean": -118.09391784667969, + "KL/rejected_KL_mean": -165.43017578125, + "KL/std": 117.8000717163086, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.0045467158779501915, + "fcm_dpo/delta": -0.05582471936941147, + "fcm_dpo/margin": 94.67250061035156, + "fcm_dpo/q_t": 0.402421772480011, + "grad_norm": 18.368188858032227, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.40962690114974976, + "logits/rejected": -0.43391185998916626, + "logps/chosen": -119.51677703857422, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -279.2939453125, + "loss": 1.096, + "margin_dpo/margin_mean": 94.67250061035156, + "margin_dpo/margin_std": 143.3657989501953, + "step": 137 + }, + { + "KL/chosen_KL_mean": -75.74452209472656, + "KL/mean": -111.02808380126953, + "KL/rejected_KL_mean": -146.31163024902344, + "KL/std": 89.46757507324219, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.005052464548498392, + "fcm_dpo/delta": 0.06267927587032318, + "fcm_dpo/margin": 70.56710815429688, + "fcm_dpo/q_t": 0.42063024640083313, + "grad_norm": 20.598432540893555, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.40584802627563477, + "logits/rejected": -0.3970094919204712, + "logps/chosen": -136.26417541503906, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -239.50857543945312, + "loss": 1.1109, + "margin_dpo/margin_mean": 70.56710815429688, + "margin_dpo/margin_std": 90.8817367553711, + "step": 138 + }, + { + "KL/chosen_KL_mean": -64.63111114501953, + "KL/mean": -105.27629089355469, + "KL/rejected_KL_mean": -145.92144775390625, + "KL/std": 83.68637084960938, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.005053876433521509, + "fcm_dpo/delta": -0.019144825637340546, + "fcm_dpo/margin": 81.29034423828125, + "fcm_dpo/q_t": 0.4056248664855957, + "grad_norm": 15.781937599182129, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.38525593280792236, + "logits/rejected": -0.37835609912872314, + "logps/chosen": -111.52249145507812, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -225.64944458007812, + "loss": 1.0888, + "margin_dpo/margin_mean": 81.29034423828125, + "margin_dpo/margin_std": 111.37606811523438, + "step": 139 + }, + { + "KL/chosen_KL_mean": -77.94660949707031, + "KL/mean": -114.49009704589844, + "KL/rejected_KL_mean": -151.03359985351562, + "KL/std": 91.72400665283203, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.0051695131696760654, + "fcm_dpo/delta": 0.03525704890489578, + "fcm_dpo/margin": 73.08699035644531, + "fcm_dpo/q_t": 0.4163084924221039, + "grad_norm": 16.439830780029297, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.3429441452026367, + "logits/rejected": -0.3306434154510498, + "logps/chosen": -136.92132568359375, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -234.3177032470703, + "loss": 1.1224, + "margin_dpo/margin_mean": 73.08699035644531, + "margin_dpo/margin_std": 112.87854766845703, + "step": 140 + }, + { + "KL/chosen_KL_mean": -88.09237670898438, + "KL/mean": -122.84248352050781, + "KL/rejected_KL_mean": -157.59259033203125, + "KL/std": 105.68617248535156, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.005516390781849623, + "fcm_dpo/delta": 0.02600482851266861, + "fcm_dpo/margin": 69.5002212524414, + "fcm_dpo/q_t": 0.4152142405509949, + "grad_norm": 25.1342716217041, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.3684391975402832, + "logits/rejected": -0.35982775688171387, + "logps/chosen": -163.16802978515625, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -255.78485107421875, + "loss": 1.1451, + "margin_dpo/margin_mean": 69.50021362304688, + "margin_dpo/margin_std": 123.44959259033203, + "step": 141 + }, + { + "KL/chosen_KL_mean": -85.69429016113281, + "KL/mean": -125.2984848022461, + "KL/rejected_KL_mean": -164.9027099609375, + "KL/std": 109.44830322265625, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.0053951190784573555, + "fcm_dpo/delta": -0.05040793865919113, + "fcm_dpo/margin": 79.20841217041016, + "fcm_dpo/q_t": 0.4017929434776306, + "grad_norm": 22.805734634399414, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.3611358404159546, + "logits/rejected": -0.36291757225990295, + "logps/chosen": -143.7222137451172, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -259.48492431640625, + "loss": 1.1263, + "margin_dpo/margin_mean": 79.20841217041016, + "margin_dpo/margin_std": 128.5096893310547, + "step": 142 + }, + { + "KL/chosen_KL_mean": -88.5622329711914, + "KL/mean": -114.20449829101562, + "KL/rejected_KL_mean": -139.84677124023438, + "KL/std": 94.44708251953125, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.005363130010664463, + "fcm_dpo/delta": 0.035532694309949875, + "fcm_dpo/margin": 51.28453063964844, + "fcm_dpo/q_t": 0.4414982795715332, + "grad_norm": 20.507484436035156, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.3294772803783417, + "logits/rejected": -0.3267552852630615, + "logps/chosen": -146.15869140625, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -218.84634399414062, + "loss": 1.2134, + "margin_dpo/margin_mean": 51.28453063964844, + "margin_dpo/margin_std": 110.63523864746094, + "step": 143 + }, + { + "KL/chosen_KL_mean": -80.96118927001953, + "KL/mean": -111.06156921386719, + "KL/rejected_KL_mean": -141.16195678710938, + "KL/std": 81.7334213256836, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.005931549705564976, + "fcm_dpo/delta": 0.0660046860575676, + "fcm_dpo/margin": 60.200782775878906, + "fcm_dpo/q_t": 0.4232863783836365, + "grad_norm": 20.33156394958496, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.3503091037273407, + "logits/rejected": -0.3469555974006653, + "logps/chosen": -140.8675537109375, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -223.1622314453125, + "loss": 1.1353, + "margin_dpo/margin_mean": 60.200782775878906, + "margin_dpo/margin_std": 95.25154113769531, + "step": 144 + }, + { + "KL/chosen_KL_mean": -77.64210510253906, + "KL/mean": -109.43412780761719, + "KL/rejected_KL_mean": -141.22616577148438, + "KL/std": 79.6153564453125, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.006183978170156479, + "fcm_dpo/delta": 0.011190323159098625, + "fcm_dpo/margin": 63.584068298339844, + "fcm_dpo/q_t": 0.41196322441101074, + "grad_norm": 25.684057235717773, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.376176118850708, + "logits/rejected": -0.3664652109146118, + "logps/chosen": -134.24276733398438, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -219.0924835205078, + "loss": 1.1163, + "margin_dpo/margin_mean": 63.584068298339844, + "margin_dpo/margin_std": 97.96441650390625, + "step": 145 + }, + { + "KL/chosen_KL_mean": -102.16140747070312, + "KL/mean": -126.88875579833984, + "KL/rejected_KL_mean": -151.61610412597656, + "KL/std": 86.07960510253906, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.006922571919858456, + "fcm_dpo/delta": 0.08818544447422028, + "fcm_dpo/margin": 49.45469665527344, + "fcm_dpo/q_t": 0.42923611402511597, + "grad_norm": 36.29349136352539, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.372928261756897, + "logits/rejected": -0.35708069801330566, + "logps/chosen": -168.161865234375, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -233.3188934326172, + "loss": 1.2057, + "margin_dpo/margin_mean": 49.45469665527344, + "margin_dpo/margin_std": 103.38996887207031, + "step": 146 + }, + { + "KL/chosen_KL_mean": -74.98482513427734, + "KL/mean": -108.55328369140625, + "KL/rejected_KL_mean": -142.12173461914062, + "KL/std": 86.59007263183594, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.006506077013909817, + "fcm_dpo/delta": -0.07235047966241837, + "fcm_dpo/margin": 67.13691711425781, + "fcm_dpo/q_t": 0.3988075852394104, + "grad_norm": 22.362390518188477, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.37692469358444214, + "logits/rejected": -0.3635402321815491, + "logps/chosen": -128.39031982421875, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -213.51235961914062, + "loss": 1.1083, + "margin_dpo/margin_mean": 67.13691711425781, + "margin_dpo/margin_std": 108.45335388183594, + "step": 147 + }, + { + "KL/chosen_KL_mean": -73.58265686035156, + "KL/mean": -103.37672424316406, + "KL/rejected_KL_mean": -133.17080688476562, + "KL/std": 88.8348388671875, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.005902706179767847, + "fcm_dpo/delta": -0.09467141330242157, + "fcm_dpo/margin": 59.5881462097168, + "fcm_dpo/q_t": 0.4176294803619385, + "grad_norm": 16.640806198120117, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.4213970899581909, + "logits/rejected": -0.4161149859428406, + "logps/chosen": -138.5197296142578, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -236.2646484375, + "loss": 1.1526, + "margin_dpo/margin_mean": 59.5881462097168, + "margin_dpo/margin_std": 94.07696533203125, + "step": 148 + }, + { + "KL/chosen_KL_mean": -68.221435546875, + "KL/mean": -104.93063354492188, + "KL/rejected_KL_mean": -141.63983154296875, + "KL/std": 77.2359619140625, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.005145592149347067, + "fcm_dpo/delta": 0.023085758090019226, + "fcm_dpo/margin": 73.41839599609375, + "fcm_dpo/q_t": 0.412888765335083, + "grad_norm": 15.609312057495117, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.43985214829444885, + "logits/rejected": -0.4274490475654602, + "logps/chosen": -126.69520568847656, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -240.95458984375, + "loss": 1.1062, + "margin_dpo/margin_mean": 73.41839599609375, + "margin_dpo/margin_std": 91.98696899414062, + "step": 149 + }, + { + "KL/chosen_KL_mean": -59.703102111816406, + "KL/mean": -95.98683166503906, + "KL/rejected_KL_mean": -132.27056884765625, + "KL/std": 89.55494689941406, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.005316386930644512, + "fcm_dpo/delta": 0.011053726077079773, + "fcm_dpo/margin": 72.56747436523438, + "fcm_dpo/q_t": 0.4142092168331146, + "grad_norm": 16.736658096313477, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.41275283694267273, + "logits/rejected": -0.41830599308013916, + "logps/chosen": -105.4089126586914, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -215.61817932128906, + "loss": 1.1153, + "margin_dpo/margin_mean": 72.56747436523438, + "margin_dpo/margin_std": 106.04312133789062, + "step": 150 + }, + { + "KL/chosen_KL_mean": -77.2176513671875, + "KL/mean": -112.65522766113281, + "KL/rejected_KL_mean": -148.09280395507812, + "KL/std": 87.22770690917969, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.005660551600158215, + "fcm_dpo/delta": -0.002181790303438902, + "fcm_dpo/margin": 70.87516784667969, + "fcm_dpo/q_t": 0.40790289640426636, + "grad_norm": 21.483774185180664, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.4776611328125, + "logits/rejected": -0.4529585838317871, + "logps/chosen": -147.78848266601562, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -248.55662536621094, + "loss": 1.0927, + "margin_dpo/margin_mean": 70.87516784667969, + "margin_dpo/margin_std": 95.74323272705078, + "step": 151 + }, + { + "KL/chosen_KL_mean": -65.65343475341797, + "KL/mean": -108.78428649902344, + "KL/rejected_KL_mean": -151.91513061523438, + "KL/std": 86.67942810058594, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.005145312752574682, + "fcm_dpo/delta": -0.07856889814138412, + "fcm_dpo/margin": 86.26168823242188, + "fcm_dpo/q_t": 0.39314860105514526, + "grad_norm": 18.017948150634766, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.4266291558742523, + "logits/rejected": -0.42508143186569214, + "logps/chosen": -125.81782531738281, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -258.05560302734375, + "loss": 1.063, + "margin_dpo/margin_mean": 86.26168823242188, + "margin_dpo/margin_std": 107.51152038574219, + "step": 152 + }, + { + "KL/chosen_KL_mean": -65.02685546875, + "KL/mean": -102.04881286621094, + "KL/rejected_KL_mean": -139.07077026367188, + "KL/std": 95.22334289550781, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.005174952559173107, + "fcm_dpo/delta": 0.024773182347416878, + "fcm_dpo/margin": 74.04390716552734, + "fcm_dpo/q_t": 0.4167134463787079, + "grad_norm": 14.81849193572998, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.44491156935691833, + "logits/rejected": -0.44107282161712646, + "logps/chosen": -121.34213256835938, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -224.72659301757812, + "loss": 1.1225, + "margin_dpo/margin_mean": 74.04390716552734, + "margin_dpo/margin_std": 117.32939147949219, + "step": 153 + }, + { + "KL/chosen_KL_mean": -78.82939910888672, + "KL/mean": -113.46821594238281, + "KL/rejected_KL_mean": -148.10702514648438, + "KL/std": 93.49275207519531, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.005523172207176685, + "fcm_dpo/delta": 0.02156968042254448, + "fcm_dpo/margin": 69.27762603759766, + "fcm_dpo/q_t": 0.41571739315986633, + "grad_norm": 18.01680564880371, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.4389742910861969, + "logits/rejected": -0.44377291202545166, + "logps/chosen": -141.57196044921875, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -252.35122680664062, + "loss": 1.1476, + "margin_dpo/margin_mean": 69.27761840820312, + "margin_dpo/margin_std": 122.24600982666016, + "step": 154 + }, + { + "KL/chosen_KL_mean": -71.37049102783203, + "KL/mean": -105.14212036132812, + "KL/rejected_KL_mean": -138.9137420654297, + "KL/std": 85.17216491699219, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.0056947702541947365, + "fcm_dpo/delta": 0.021101046353578568, + "fcm_dpo/margin": 67.54327392578125, + "fcm_dpo/q_t": 0.41405290365219116, + "grad_norm": 18.612998962402344, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.45382189750671387, + "logits/rejected": -0.43611928820610046, + "logps/chosen": -132.023681640625, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -216.40594482421875, + "loss": 1.1265, + "margin_dpo/margin_mean": 67.54327392578125, + "margin_dpo/margin_std": 108.4144287109375, + "step": 155 + }, + { + "KL/chosen_KL_mean": -96.85684204101562, + "KL/mean": -116.4049301147461, + "KL/rejected_KL_mean": -135.95301818847656, + "KL/std": 89.89227294921875, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.006321663036942482, + "fcm_dpo/delta": 0.06302600353956223, + "fcm_dpo/margin": 39.09617614746094, + "fcm_dpo/q_t": 0.44758695363998413, + "grad_norm": 30.659738540649414, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.4037202298641205, + "logits/rejected": -0.3993948698043823, + "logps/chosen": -166.34872436523438, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -213.122314453125, + "loss": 1.2862, + "margin_dpo/margin_mean": 39.09617614746094, + "margin_dpo/margin_std": 119.15692138671875, + "step": 156 + }, + { + "KL/chosen_KL_mean": -85.81370544433594, + "KL/mean": -130.08242797851562, + "KL/rejected_KL_mean": -174.35113525390625, + "KL/std": 98.00643920898438, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.004979247227311134, + "fcm_dpo/delta": -0.10685013234615326, + "fcm_dpo/margin": 88.53744506835938, + "fcm_dpo/q_t": 0.38473406434059143, + "grad_norm": 23.637866973876953, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.37596505880355835, + "logits/rejected": -0.38393884897232056, + "logps/chosen": -147.18214416503906, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -281.99749755859375, + "loss": 1.073, + "margin_dpo/margin_mean": 88.53744506835938, + "margin_dpo/margin_std": 110.94694519042969, + "step": 157 + }, + { + "KL/chosen_KL_mean": -86.83956909179688, + "KL/mean": -135.0098876953125, + "KL/rejected_KL_mean": -183.18020629882812, + "KL/std": 121.85169982910156, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.004572378005832434, + "fcm_dpo/delta": -0.07351740449666977, + "fcm_dpo/margin": 96.34066009521484, + "fcm_dpo/q_t": 0.3994863033294678, + "grad_norm": 16.999359130859375, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.37230780720710754, + "logits/rejected": -0.391128271818161, + "logps/chosen": -144.45248413085938, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -296.8748779296875, + "loss": 1.0944, + "margin_dpo/margin_mean": 96.34065246582031, + "margin_dpo/margin_std": 147.7144775390625, + "step": 158 + }, + { + "KL/chosen_KL_mean": -100.27496337890625, + "KL/mean": -131.74691772460938, + "KL/rejected_KL_mean": -163.2188720703125, + "KL/std": 107.50895690917969, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.004534922074526548, + "fcm_dpo/delta": 0.02816414088010788, + "fcm_dpo/margin": 62.94389343261719, + "fcm_dpo/q_t": 0.435373455286026, + "grad_norm": 18.136377334594727, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.3773287534713745, + "logits/rejected": -0.34688425064086914, + "logps/chosen": -181.83531188964844, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -252.11758422851562, + "loss": 1.1876, + "margin_dpo/margin_mean": 62.94389343261719, + "margin_dpo/margin_std": 117.23402404785156, + "step": 159 + }, + { + "KL/chosen_KL_mean": -103.02062225341797, + "KL/mean": -146.32418823242188, + "KL/rejected_KL_mean": -189.62774658203125, + "KL/std": 114.74168395996094, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.0045206244103610516, + "fcm_dpo/delta": 0.009057410061359406, + "fcm_dpo/margin": 86.60713958740234, + "fcm_dpo/q_t": 0.4141198396682739, + "grad_norm": 18.559383392333984, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.3365239202976227, + "logits/rejected": -0.3285549283027649, + "logps/chosen": -168.75149536132812, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -286.8455810546875, + "loss": 1.1313, + "margin_dpo/margin_mean": 86.60713958740234, + "margin_dpo/margin_std": 143.93826293945312, + "step": 160 + }, + { + "KL/chosen_KL_mean": -87.73458862304688, + "KL/mean": -126.02169799804688, + "KL/rejected_KL_mean": -164.30880737304688, + "KL/std": 91.14512634277344, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.004974995274096727, + "fcm_dpo/delta": 0.030578553676605225, + "fcm_dpo/margin": 76.57421875, + "fcm_dpo/q_t": 0.41616570949554443, + "grad_norm": 17.85219383239746, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.34718695282936096, + "logits/rejected": -0.34764522314071655, + "logps/chosen": -140.17105102539062, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -247.73976135253906, + "loss": 1.1167, + "margin_dpo/margin_mean": 76.57421875, + "margin_dpo/margin_std": 114.59629821777344, + "step": 161 + }, + { + "KL/chosen_KL_mean": -87.06214904785156, + "KL/mean": -124.53192901611328, + "KL/rejected_KL_mean": -162.001708984375, + "KL/std": 101.81707763671875, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.004378783982247114, + "fcm_dpo/delta": -0.07103817164897919, + "fcm_dpo/margin": 74.93955993652344, + "fcm_dpo/q_t": 0.41890764236450195, + "grad_norm": 17.84354019165039, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.40842747688293457, + "logits/rejected": -0.3954179883003235, + "logps/chosen": -149.6727294921875, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -251.3922882080078, + "loss": 1.1551, + "margin_dpo/margin_mean": 74.93955993652344, + "margin_dpo/margin_std": 121.97283935546875, + "step": 162 + }, + { + "KL/chosen_KL_mean": -97.13140869140625, + "KL/mean": -133.43283081054688, + "KL/rejected_KL_mean": -169.73428344726562, + "KL/std": 102.79348754882812, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.004893934354186058, + "fcm_dpo/delta": 0.06739577651023865, + "fcm_dpo/margin": 72.60286712646484, + "fcm_dpo/q_t": 0.4250032901763916, + "grad_norm": 18.780895233154297, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.2808549404144287, + "logits/rejected": -0.2610132694244385, + "logps/chosen": -152.15304565429688, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -245.15249633789062, + "loss": 1.1469, + "margin_dpo/margin_mean": 72.60285949707031, + "margin_dpo/margin_std": 123.78816223144531, + "step": 163 + }, + { + "KL/chosen_KL_mean": -90.34315490722656, + "KL/mean": -135.48098754882812, + "KL/rejected_KL_mean": -180.6188201904297, + "KL/std": 100.77694702148438, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.004474318120628595, + "fcm_dpo/delta": -0.025637470185756683, + "fcm_dpo/margin": 90.27565002441406, + "fcm_dpo/q_t": 0.40300068259239197, + "grad_norm": 18.992538452148438, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.33539408445358276, + "logits/rejected": -0.32457149028778076, + "logps/chosen": -145.98382568359375, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -260.283447265625, + "loss": 1.0914, + "margin_dpo/margin_mean": 90.27565002441406, + "margin_dpo/margin_std": 116.16989135742188, + "step": 164 + }, + { + "KL/chosen_KL_mean": -97.24948120117188, + "KL/mean": -127.26203155517578, + "KL/rejected_KL_mean": -157.2745819091797, + "KL/std": 88.08245849609375, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.005466942209750414, + "fcm_dpo/delta": 0.10794621706008911, + "fcm_dpo/margin": 60.025089263916016, + "fcm_dpo/q_t": 0.43372809886932373, + "grad_norm": 26.800586700439453, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.3530671000480652, + "logits/rejected": -0.3305925726890564, + "logps/chosen": -158.5601806640625, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -230.9451904296875, + "loss": 1.1889, + "margin_dpo/margin_mean": 60.025089263916016, + "margin_dpo/margin_std": 118.67210388183594, + "step": 165 + }, + { + "KL/chosen_KL_mean": -87.03776550292969, + "KL/mean": -144.3040771484375, + "KL/rejected_KL_mean": -201.57040405273438, + "KL/std": 130.58055114746094, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.004105499014258385, + "fcm_dpo/delta": -0.187312051653862, + "fcm_dpo/margin": 114.53263854980469, + "fcm_dpo/q_t": 0.3861845135688782, + "grad_norm": 15.165700912475586, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.2735704779624939, + "logits/rejected": -0.2837375998497009, + "logps/chosen": -138.0213623046875, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -299.66552734375, + "loss": 1.0692, + "margin_dpo/margin_mean": 114.53263854980469, + "margin_dpo/margin_std": 165.4405059814453, + "step": 166 + }, + { + "KL/chosen_KL_mean": -88.03388977050781, + "KL/mean": -147.37371826171875, + "KL/rejected_KL_mean": -206.71356201171875, + "KL/std": 116.15589904785156, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.0036684987135231495, + "fcm_dpo/delta": -0.06418722122907639, + "fcm_dpo/margin": 118.67967987060547, + "fcm_dpo/q_t": 0.3941141664981842, + "grad_norm": 17.225709915161133, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.2889915704727173, + "logits/rejected": -0.29361122846603394, + "logps/chosen": -138.45797729492188, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -302.7439880371094, + "loss": 1.0561, + "margin_dpo/margin_mean": 118.6796875, + "margin_dpo/margin_std": 136.93655395507812, + "step": 167 + }, + { + "KL/chosen_KL_mean": -97.47393798828125, + "KL/mean": -140.19537353515625, + "KL/rejected_KL_mean": -182.9167938232422, + "KL/std": 109.24185180664062, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.004024019930511713, + "fcm_dpo/delta": 0.08574581891298294, + "fcm_dpo/margin": 85.44287109375, + "fcm_dpo/q_t": 0.4270554780960083, + "grad_norm": 18.420507431030273, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.29539740085601807, + "logits/rejected": -0.2888868451118469, + "logps/chosen": -146.936767578125, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -258.225341796875, + "loss": 1.1323, + "margin_dpo/margin_mean": 85.44287109375, + "margin_dpo/margin_std": 124.32752990722656, + "step": 168 + }, + { + "KL/chosen_KL_mean": -101.9376449584961, + "KL/mean": -139.52291870117188, + "KL/rejected_KL_mean": -177.1082000732422, + "KL/std": 107.82769012451172, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.0039043950382620096, + "fcm_dpo/delta": -0.04128513112664223, + "fcm_dpo/margin": 75.17054748535156, + "fcm_dpo/q_t": 0.4314765930175781, + "grad_norm": 17.273651123046875, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.3079976439476013, + "logits/rejected": -0.2939789295196533, + "logps/chosen": -161.7410888671875, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -260.4539489746094, + "loss": 1.1953, + "margin_dpo/margin_mean": 75.17054748535156, + "margin_dpo/margin_std": 148.5015869140625, + "step": 169 + }, + { + "KL/chosen_KL_mean": -92.11761474609375, + "KL/mean": -135.65200805664062, + "KL/rejected_KL_mean": -179.18641662597656, + "KL/std": 106.36629486083984, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.004083374515175819, + "fcm_dpo/delta": 0.06564062833786011, + "fcm_dpo/margin": 87.06880187988281, + "fcm_dpo/q_t": 0.422102689743042, + "grad_norm": 16.470184326171875, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.2703856825828552, + "logits/rejected": -0.26214104890823364, + "logps/chosen": -141.58938598632812, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -255.103759765625, + "loss": 1.1365, + "margin_dpo/margin_mean": 87.06879425048828, + "margin_dpo/margin_std": 128.26217651367188, + "step": 170 + }, + { + "KL/chosen_KL_mean": -134.2930908203125, + "KL/mean": -169.67434692382812, + "KL/rejected_KL_mean": -205.0555877685547, + "KL/std": 124.128662109375, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.00497001688927412, + "fcm_dpo/delta": 0.07393293082714081, + "fcm_dpo/margin": 70.76249694824219, + "fcm_dpo/q_t": 0.4315279722213745, + "grad_norm": 27.584213256835938, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.29333311319351196, + "logits/rejected": -0.2810017764568329, + "logps/chosen": -218.7924041748047, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -314.43768310546875, + "loss": 1.2014, + "margin_dpo/margin_mean": 70.76250457763672, + "margin_dpo/margin_std": 159.743896484375, + "step": 171 + }, + { + "KL/chosen_KL_mean": -115.03340148925781, + "KL/mean": -152.57827758789062, + "KL/rejected_KL_mean": -190.12313842773438, + "KL/std": 118.62234497070312, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.0052231112495064735, + "fcm_dpo/delta": 0.01281255204230547, + "fcm_dpo/margin": 75.0897445678711, + "fcm_dpo/q_t": 0.4145969748497009, + "grad_norm": 22.732559204101562, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.3138948976993561, + "logits/rejected": -0.29758453369140625, + "logps/chosen": -183.68731689453125, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -275.559814453125, + "loss": 1.1405, + "margin_dpo/margin_mean": 75.08975219726562, + "margin_dpo/margin_std": 133.15069580078125, + "step": 172 + }, + { + "KL/chosen_KL_mean": -104.79826354980469, + "KL/mean": -142.23779296875, + "KL/rejected_KL_mean": -179.67730712890625, + "KL/std": 111.67139434814453, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.00543089909479022, + "fcm_dpo/delta": -0.016082536429166794, + "fcm_dpo/margin": 74.8790512084961, + "fcm_dpo/q_t": 0.40886810421943665, + "grad_norm": 25.01607322692871, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.377377450466156, + "logits/rejected": -0.3528056740760803, + "logps/chosen": -167.84915161132812, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -258.3612365722656, + "loss": 1.1119, + "margin_dpo/margin_mean": 74.8790512084961, + "margin_dpo/margin_std": 115.37376403808594, + "step": 173 + }, + { + "KL/chosen_KL_mean": -96.8199462890625, + "KL/mean": -141.00047302246094, + "KL/rejected_KL_mean": -185.18099975585938, + "KL/std": 111.30195617675781, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.0049737137742340565, + "fcm_dpo/delta": -0.0776623785495758, + "fcm_dpo/margin": 88.36105346679688, + "fcm_dpo/q_t": 0.3986249268054962, + "grad_norm": 24.379945755004883, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.30099165439605713, + "logits/rejected": -0.3199496269226074, + "logps/chosen": -150.18292236328125, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -287.09222412109375, + "loss": 1.1075, + "margin_dpo/margin_mean": 88.36104583740234, + "margin_dpo/margin_std": 135.80799865722656, + "step": 174 + }, + { + "KL/chosen_KL_mean": -88.39326477050781, + "KL/mean": -147.26588439941406, + "KL/rejected_KL_mean": -206.13851928710938, + "KL/std": 116.80426788330078, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.0037730231415480375, + "fcm_dpo/delta": -0.0859874039888382, + "fcm_dpo/margin": 117.74525451660156, + "fcm_dpo/q_t": 0.3884362578392029, + "grad_norm": 26.734390258789062, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.3031741976737976, + "logits/rejected": -0.28989243507385254, + "logps/chosen": -133.81103515625, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -295.6443176269531, + "loss": 1.0409, + "margin_dpo/margin_mean": 117.74525451660156, + "margin_dpo/margin_std": 123.38618469238281, + "step": 175 + }, + { + "KL/chosen_KL_mean": -90.31321716308594, + "KL/mean": -141.83433532714844, + "KL/rejected_KL_mean": -193.35546875, + "KL/std": 116.11126708984375, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.0038136134389787912, + "fcm_dpo/delta": 0.010705736465752125, + "fcm_dpo/margin": 103.042236328125, + "fcm_dpo/q_t": 0.4118039608001709, + "grad_norm": 16.391658782958984, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.309362530708313, + "logits/rejected": -0.325575590133667, + "logps/chosen": -140.76605224609375, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -288.9144287109375, + "loss": 1.0923, + "margin_dpo/margin_mean": 103.042236328125, + "margin_dpo/margin_std": 134.9136199951172, + "step": 176 + }, + { + "KL/chosen_KL_mean": -105.74380493164062, + "KL/mean": -156.05882263183594, + "KL/rejected_KL_mean": -206.3738250732422, + "KL/std": 128.35159301757812, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.003937173634767532, + "fcm_dpo/delta": 0.002443891018629074, + "fcm_dpo/margin": 100.63003540039062, + "fcm_dpo/q_t": 0.41030067205429077, + "grad_norm": 20.429180145263672, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.3264981508255005, + "logits/rejected": -0.31806203722953796, + "logps/chosen": -166.96026611328125, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -302.2676086425781, + "loss": 1.0932, + "margin_dpo/margin_mean": 100.63003540039062, + "margin_dpo/margin_std": 134.44273376464844, + "step": 177 + }, + { + "KL/chosen_KL_mean": -116.06280517578125, + "KL/mean": -179.25152587890625, + "KL/rejected_KL_mean": -242.44021606445312, + "KL/std": 147.4817657470703, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.0033705285750329494, + "fcm_dpo/delta": -0.053363341838121414, + "fcm_dpo/margin": 126.37743377685547, + "fcm_dpo/q_t": 0.39890462160110474, + "grad_norm": 19.848587036132812, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.22271186113357544, + "logits/rejected": -0.21497488021850586, + "logps/chosen": -174.3275909423828, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -347.8055419921875, + "loss": 1.0721, + "margin_dpo/margin_mean": 126.37743377685547, + "margin_dpo/margin_std": 161.1014862060547, + "step": 178 + }, + { + "KL/chosen_KL_mean": -124.49735260009766, + "KL/mean": -168.00872802734375, + "KL/rejected_KL_mean": -211.52008056640625, + "KL/std": 130.20875549316406, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.003844078164547682, + "fcm_dpo/delta": 0.0992623046040535, + "fcm_dpo/margin": 87.0227279663086, + "fcm_dpo/q_t": 0.43123751878738403, + "grad_norm": 34.565582275390625, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.27690017223358154, + "logits/rejected": -0.2880879044532776, + "logps/chosen": -185.55567932128906, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -302.04791259765625, + "loss": 1.1778, + "margin_dpo/margin_mean": 87.0227279663086, + "margin_dpo/margin_std": 163.81394958496094, + "step": 179 + }, + { + "KL/chosen_KL_mean": -107.85954284667969, + "KL/mean": -166.43894958496094, + "KL/rejected_KL_mean": -225.0183563232422, + "KL/std": 115.33984375, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.0037114559672772884, + "fcm_dpo/delta": -0.06508003920316696, + "fcm_dpo/margin": 117.1588134765625, + "fcm_dpo/q_t": 0.39406827092170715, + "grad_norm": 16.350337982177734, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.23366385698318481, + "logits/rejected": -0.22989457845687866, + "logps/chosen": -162.2022705078125, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -323.2301940917969, + "loss": 1.0637, + "margin_dpo/margin_mean": 117.1588134765625, + "margin_dpo/margin_std": 143.82162475585938, + "step": 180 + }, + { + "KL/chosen_KL_mean": -94.5392074584961, + "KL/mean": -129.8724365234375, + "KL/rejected_KL_mean": -165.2056884765625, + "KL/std": 107.95272827148438, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.004456082358956337, + "fcm_dpo/delta": 0.11765934526920319, + "fcm_dpo/margin": 70.6664810180664, + "fcm_dpo/q_t": 0.4351644515991211, + "grad_norm": 19.006668090820312, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.2771533727645874, + "logits/rejected": -0.2554428279399872, + "logps/chosen": -149.5396728515625, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -226.86184692382812, + "loss": 1.1826, + "margin_dpo/margin_mean": 70.6664810180664, + "margin_dpo/margin_std": 136.99830627441406, + "step": 181 + }, + { + "KL/chosen_KL_mean": -89.14537811279297, + "KL/mean": -153.301513671875, + "KL/rejected_KL_mean": -217.45767211914062, + "KL/std": 129.44326782226562, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.0035601062700152397, + "fcm_dpo/delta": -0.1331714242696762, + "fcm_dpo/margin": 128.31228637695312, + "fcm_dpo/q_t": 0.38162392377853394, + "grad_norm": 15.877484321594238, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.24869897961616516, + "logits/rejected": -0.259911447763443, + "logps/chosen": -130.25323486328125, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -306.979248046875, + "loss": 1.054, + "margin_dpo/margin_mean": 128.31228637695312, + "margin_dpo/margin_std": 157.36795043945312, + "step": 182 + }, + { + "KL/chosen_KL_mean": -132.1239776611328, + "KL/mean": -164.78997802734375, + "KL/rejected_KL_mean": -197.4559783935547, + "KL/std": 107.94178771972656, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.003544195555150509, + "fcm_dpo/delta": 0.000493988860398531, + "fcm_dpo/margin": 65.33200073242188, + "fcm_dpo/q_t": 0.4455084502696991, + "grad_norm": 17.170263290405273, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.2581177055835724, + "logits/rejected": -0.2515699863433838, + "logps/chosen": -189.64854431152344, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -273.43170166015625, + "loss": 1.2172, + "margin_dpo/margin_mean": 65.33200073242188, + "margin_dpo/margin_std": 118.64833068847656, + "step": 183 + }, + { + "KL/chosen_KL_mean": -111.37728881835938, + "KL/mean": -146.77032470703125, + "KL/rejected_KL_mean": -182.16336059570312, + "KL/std": 101.87884521484375, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.003663355251774192, + "fcm_dpo/delta": 0.03253895044326782, + "fcm_dpo/margin": 70.78607177734375, + "fcm_dpo/q_t": 0.4417157769203186, + "grad_norm": 16.201236724853516, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.28559648990631104, + "logits/rejected": -0.27648642659187317, + "logps/chosen": -169.9222412109375, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -258.79742431640625, + "loss": 1.1955, + "margin_dpo/margin_mean": 70.78607177734375, + "margin_dpo/margin_std": 128.29718017578125, + "step": 184 + }, + { + "KL/chosen_KL_mean": -122.75660705566406, + "KL/mean": -149.69287109375, + "KL/rejected_KL_mean": -176.62911987304688, + "KL/std": 117.6004638671875, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.003782514948397875, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 53.87251663208008, + "fcm_dpo/q_t": 0.4551408886909485, + "grad_norm": 18.46157455444336, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.2715925872325897, + "logits/rejected": -0.25268396735191345, + "logps/chosen": -184.78245544433594, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -250.3917236328125, + "loss": 1.2595, + "margin_dpo/margin_mean": 53.872520446777344, + "margin_dpo/margin_std": 141.81597900390625, + "step": 185 + }, + { + "KL/chosen_KL_mean": -115.03028869628906, + "KL/mean": -167.48983764648438, + "KL/rejected_KL_mean": -219.94937133789062, + "KL/std": 118.58721160888672, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.0037781421560794115, + "fcm_dpo/delta": 0.005793063435703516, + "fcm_dpo/margin": 104.91907501220703, + "fcm_dpo/q_t": 0.40997469425201416, + "grad_norm": 19.549184799194336, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.3371797502040863, + "logits/rejected": -0.3270975947380066, + "logps/chosen": -184.38375854492188, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -308.0218200683594, + "loss": 1.0931, + "margin_dpo/margin_mean": 104.9190673828125, + "margin_dpo/margin_std": 139.93943786621094, + "step": 186 + }, + { + "KL/chosen_KL_mean": -102.89697265625, + "KL/mean": -148.2735137939453, + "KL/rejected_KL_mean": -193.65003967285156, + "KL/std": 112.91288757324219, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.00417533703148365, + "fcm_dpo/delta": 0.025836054235696793, + "fcm_dpo/margin": 90.75306701660156, + "fcm_dpo/q_t": 0.4136863350868225, + "grad_norm": 21.79146385192871, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.2833201587200165, + "logits/rejected": -0.27770209312438965, + "logps/chosen": -155.6534423828125, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -275.619140625, + "loss": 1.1064, + "margin_dpo/margin_mean": 90.75306701660156, + "margin_dpo/margin_std": 121.31842041015625, + "step": 187 + }, + { + "KL/chosen_KL_mean": -96.85295104980469, + "KL/mean": -152.17840576171875, + "KL/rejected_KL_mean": -207.50384521484375, + "KL/std": 124.83462524414062, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.003823988139629364, + "fcm_dpo/delta": -0.04001890495419502, + "fcm_dpo/margin": 110.6509017944336, + "fcm_dpo/q_t": 0.4005782902240753, + "grad_norm": 25.183116912841797, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.2504292130470276, + "logits/rejected": -0.2585201859474182, + "logps/chosen": -146.2684326171875, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -297.04425048828125, + "loss": 1.0712, + "margin_dpo/margin_mean": 110.6509017944336, + "margin_dpo/margin_std": 138.2140655517578, + "step": 188 + }, + { + "KL/chosen_KL_mean": -115.58564758300781, + "KL/mean": -160.01602172851562, + "KL/rejected_KL_mean": -204.44639587402344, + "KL/std": 127.6397933959961, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.004090734291821718, + "fcm_dpo/delta": 0.055509161204099655, + "fcm_dpo/margin": 88.86076354980469, + "fcm_dpo/q_t": 0.42382127046585083, + "grad_norm": 27.4331111907959, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.271611750125885, + "logits/rejected": -0.2570100426673889, + "logps/chosen": -167.98460388183594, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -276.6137390136719, + "loss": 1.1472, + "margin_dpo/margin_mean": 88.86076354980469, + "margin_dpo/margin_std": 158.424560546875, + "step": 189 + }, + { + "KL/chosen_KL_mean": -124.001708984375, + "KL/mean": -178.27090454101562, + "KL/rejected_KL_mean": -232.54010009765625, + "KL/std": 137.54371643066406, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.004083447623997927, + "fcm_dpo/delta": -0.09416892379522324, + "fcm_dpo/margin": 108.53840637207031, + "fcm_dpo/q_t": 0.3950834274291992, + "grad_norm": 19.966215133666992, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.3038903474807739, + "logits/rejected": -0.29708611965179443, + "logps/chosen": -188.68478393554688, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -335.09063720703125, + "loss": 1.1112, + "margin_dpo/margin_mean": 108.53840637207031, + "margin_dpo/margin_std": 159.21173095703125, + "step": 190 + }, + { + "KL/chosen_KL_mean": -116.2418212890625, + "KL/mean": -192.55517578125, + "KL/rejected_KL_mean": -268.8685302734375, + "KL/std": 155.87002563476562, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.0028747329488396645, + "fcm_dpo/delta": -0.08564233034849167, + "fcm_dpo/margin": 152.626708984375, + "fcm_dpo/q_t": 0.3907443881034851, + "grad_norm": 17.792726516723633, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.2973872721195221, + "logits/rejected": -0.2730400264263153, + "logps/chosen": -184.90069580078125, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -379.0082092285156, + "loss": 1.0538, + "margin_dpo/margin_mean": 152.626708984375, + "margin_dpo/margin_std": 175.80136108398438, + "step": 191 + }, + { + "KL/chosen_KL_mean": -149.68504333496094, + "KL/mean": -200.70745849609375, + "KL/rejected_KL_mean": -251.7298583984375, + "KL/std": 139.31866455078125, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.003366319462656975, + "fcm_dpo/delta": 0.079339399933815, + "fcm_dpo/margin": 102.04479217529297, + "fcm_dpo/q_t": 0.42711859941482544, + "grad_norm": 20.006580352783203, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.2600586414337158, + "logits/rejected": -0.26135045289993286, + "logps/chosen": -219.41195678710938, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -355.05120849609375, + "loss": 1.1503, + "margin_dpo/margin_mean": 102.0447998046875, + "margin_dpo/margin_std": 174.04171752929688, + "step": 192 + }, + { + "KL/chosen_KL_mean": -149.03636169433594, + "KL/mean": -177.56857299804688, + "KL/rejected_KL_mean": -206.10079956054688, + "KL/std": 129.29698181152344, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.003643119242042303, + "fcm_dpo/delta": 0.06691880524158478, + "fcm_dpo/margin": 57.06443786621094, + "fcm_dpo/q_t": 0.45318782329559326, + "grad_norm": 28.55337142944336, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.27016884088516235, + "logits/rejected": -0.2688116431236267, + "logps/chosen": -209.22686767578125, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -282.50836181640625, + "loss": 1.2736, + "margin_dpo/margin_mean": 57.06443786621094, + "margin_dpo/margin_std": 165.7989501953125, + "step": 193 + }, + { + "KL/chosen_KL_mean": -90.87747955322266, + "KL/mean": -138.6411590576172, + "KL/rejected_KL_mean": -186.40484619140625, + "KL/std": 105.31715393066406, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.0039482079446315765, + "fcm_dpo/delta": 0.034811198711395264, + "fcm_dpo/margin": 95.52735900878906, + "fcm_dpo/q_t": 0.4142157733440399, + "grad_norm": 17.623666763305664, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.1908482015132904, + "logits/rejected": -0.17822426557540894, + "logps/chosen": -128.7178497314453, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -247.08963012695312, + "loss": 1.0982, + "margin_dpo/margin_mean": 95.52735900878906, + "margin_dpo/margin_std": 116.02978515625, + "step": 194 + }, + { + "KL/chosen_KL_mean": -144.2040557861328, + "KL/mean": -196.9357147216797, + "KL/rejected_KL_mean": -249.66738891601562, + "KL/std": 133.17889404296875, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.0039472421631217, + "fcm_dpo/delta": -0.028900716453790665, + "fcm_dpo/margin": 105.46333312988281, + "fcm_dpo/q_t": 0.40308016538619995, + "grad_norm": 22.52235984802246, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.23734885454177856, + "logits/rejected": -0.25579947233200073, + "logps/chosen": -199.0956268310547, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -346.4383544921875, + "loss": 1.0821, + "margin_dpo/margin_mean": 105.46334075927734, + "margin_dpo/margin_std": 139.16427612304688, + "step": 195 + }, + { + "KL/chosen_KL_mean": -111.86429595947266, + "KL/mean": -169.64901733398438, + "KL/rejected_KL_mean": -227.4337158203125, + "KL/std": 129.91050720214844, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.003521410282701254, + "fcm_dpo/delta": -0.028155531734228134, + "fcm_dpo/margin": 115.56942749023438, + "fcm_dpo/q_t": 0.40525445342063904, + "grad_norm": 16.956315994262695, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.24712811410427094, + "logits/rejected": -0.2348405420780182, + "logps/chosen": -165.1095428466797, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -303.4866638183594, + "loss": 1.0964, + "margin_dpo/margin_mean": 115.56942749023438, + "margin_dpo/margin_std": 156.6468505859375, + "step": 196 + }, + { + "KL/chosen_KL_mean": -114.23474884033203, + "KL/mean": -157.83114624023438, + "KL/rejected_KL_mean": -201.42755126953125, + "KL/std": 111.18183135986328, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.004171357490122318, + "fcm_dpo/delta": 0.04900194704532623, + "fcm_dpo/margin": 87.19279479980469, + "fcm_dpo/q_t": 0.4201440215110779, + "grad_norm": 31.859424591064453, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.24368175864219666, + "logits/rejected": -0.24027302861213684, + "logps/chosen": -174.65509033203125, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -278.636474609375, + "loss": 1.1232, + "margin_dpo/margin_mean": 87.19279479980469, + "margin_dpo/margin_std": 128.5411376953125, + "step": 197 + }, + { + "KL/chosen_KL_mean": -124.44345092773438, + "KL/mean": -183.01571655273438, + "KL/rejected_KL_mean": -241.5879669189453, + "KL/std": 141.48675537109375, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.003705207258462906, + "fcm_dpo/delta": -0.06200866401195526, + "fcm_dpo/margin": 117.14451599121094, + "fcm_dpo/q_t": 0.39838171005249023, + "grad_norm": 22.00115966796875, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.29106560349464417, + "logits/rejected": -0.2917174696922302, + "logps/chosen": -179.47964477539062, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -338.83123779296875, + "loss": 1.0749, + "margin_dpo/margin_mean": 117.14451599121094, + "margin_dpo/margin_std": 159.01614379882812, + "step": 198 + }, + { + "KL/chosen_KL_mean": -120.00575256347656, + "KL/mean": -176.61056518554688, + "KL/rejected_KL_mean": -233.21536254882812, + "KL/std": 125.2437515258789, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.00345216179266572, + "fcm_dpo/delta": 0.006332229822874069, + "fcm_dpo/margin": 113.2095947265625, + "fcm_dpo/q_t": 0.4100239872932434, + "grad_norm": 18.751543045043945, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.2945285141468048, + "logits/rejected": -0.2768348157405853, + "logps/chosen": -176.8345947265625, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -317.86358642578125, + "loss": 1.106, + "margin_dpo/margin_mean": 113.2095947265625, + "margin_dpo/margin_std": 155.6266632080078, + "step": 199 + }, + { + "KL/chosen_KL_mean": -117.3885498046875, + "KL/mean": -176.23748779296875, + "KL/rejected_KL_mean": -235.08642578125, + "KL/std": 138.85617065429688, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.0034625413827598095, + "fcm_dpo/delta": -0.016614485532045364, + "fcm_dpo/margin": 117.6978988647461, + "fcm_dpo/q_t": 0.4070635437965393, + "grad_norm": 20.583110809326172, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.2415538728237152, + "logits/rejected": -0.22881929576396942, + "logps/chosen": -170.4556121826172, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -315.69488525390625, + "loss": 1.1079, + "margin_dpo/margin_mean": 117.6978988647461, + "margin_dpo/margin_std": 178.77822875976562, + "step": 200 + }, + { + "KL/chosen_KL_mean": -128.0514373779297, + "KL/mean": -184.71365356445312, + "KL/rejected_KL_mean": -241.3758544921875, + "KL/std": 148.18923950195312, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.00359878852032125, + "fcm_dpo/delta": -0.015944721177220345, + "fcm_dpo/margin": 113.32442474365234, + "fcm_dpo/q_t": 0.40622514486312866, + "grad_norm": 24.77713966369629, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.3444761037826538, + "logits/rejected": -0.3457464575767517, + "logps/chosen": -203.4536590576172, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -356.18408203125, + "loss": 1.1082, + "margin_dpo/margin_mean": 113.32442474365234, + "margin_dpo/margin_std": 168.1522979736328, + "step": 201 + }, + { + "KL/chosen_KL_mean": -129.18331909179688, + "KL/mean": -168.78958129882812, + "KL/rejected_KL_mean": -208.39584350585938, + "KL/std": 127.8756103515625, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.004097277298569679, + "fcm_dpo/delta": 0.10765650868415833, + "fcm_dpo/margin": 79.21253204345703, + "fcm_dpo/q_t": 0.4345342516899109, + "grad_norm": 22.265583038330078, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.27213054895401, + "logits/rejected": -0.28951022028923035, + "logps/chosen": -179.28463745117188, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -295.380859375, + "loss": 1.1947, + "margin_dpo/margin_mean": 79.21253967285156, + "margin_dpo/margin_std": 168.17898559570312, + "step": 202 + }, + { + "KL/chosen_KL_mean": -128.6010284423828, + "KL/mean": -178.1983642578125, + "KL/rejected_KL_mean": -227.79568481445312, + "KL/std": 133.08248901367188, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.004127629101276398, + "fcm_dpo/delta": -0.01639389619231224, + "fcm_dpo/margin": 99.19467163085938, + "fcm_dpo/q_t": 0.40751299262046814, + "grad_norm": 20.381248474121094, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.303178995847702, + "logits/rejected": -0.29665249586105347, + "logps/chosen": -189.21072387695312, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -313.691650390625, + "loss": 1.1046, + "margin_dpo/margin_mean": 99.19467163085938, + "margin_dpo/margin_std": 150.01708984375, + "step": 203 + }, + { + "KL/chosen_KL_mean": -141.3732452392578, + "KL/mean": -185.82644653320312, + "KL/rejected_KL_mean": -230.27967834472656, + "KL/std": 142.70265197753906, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.0043134503066539764, + "fcm_dpo/delta": 0.026382368057966232, + "fcm_dpo/margin": 88.90643310546875, + "fcm_dpo/q_t": 0.4196096360683441, + "grad_norm": 31.27920150756836, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.38292059302330017, + "logits/rejected": -0.34853625297546387, + "logps/chosen": -221.5382080078125, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -317.9755859375, + "loss": 1.1628, + "margin_dpo/margin_mean": 88.90642547607422, + "margin_dpo/margin_std": 170.709228515625, + "step": 204 + }, + { + "KL/chosen_KL_mean": -131.44627380371094, + "KL/mean": -190.60824584960938, + "KL/rejected_KL_mean": -249.77024841308594, + "KL/std": 140.37823486328125, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.0035104253329336643, + "fcm_dpo/delta": -0.07186352461576462, + "fcm_dpo/margin": 118.3239517211914, + "fcm_dpo/q_t": 0.3940643072128296, + "grad_norm": 19.9980411529541, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.290398508310318, + "logits/rejected": -0.2605670094490051, + "logps/chosen": -190.8310089111328, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -334.8952941894531, + "loss": 1.0949, + "margin_dpo/margin_mean": 118.3239517211914, + "margin_dpo/margin_std": 153.96115112304688, + "step": 205 + }, + { + "KL/chosen_KL_mean": -112.48219299316406, + "KL/mean": -175.43797302246094, + "KL/rejected_KL_mean": -238.3937530517578, + "KL/std": 123.48064422607422, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.0033509135246276855, + "fcm_dpo/delta": -0.047678425908088684, + "fcm_dpo/margin": 125.91154479980469, + "fcm_dpo/q_t": 0.39560678601264954, + "grad_norm": 30.06896209716797, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.26723650097846985, + "logits/rejected": -0.2706051170825958, + "logps/chosen": -159.44668579101562, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -337.34722900390625, + "loss": 1.0582, + "margin_dpo/margin_mean": 125.91154479980469, + "margin_dpo/margin_std": 135.0882568359375, + "step": 206 + }, + { + "KL/chosen_KL_mean": -106.53071594238281, + "KL/mean": -176.65072631835938, + "KL/rejected_KL_mean": -246.77072143554688, + "KL/std": 144.35787963867188, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.0031110041309148073, + "fcm_dpo/delta": -0.06504091620445251, + "fcm_dpo/margin": 140.24000549316406, + "fcm_dpo/q_t": 0.3961068391799927, + "grad_norm": 30.773921966552734, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.35365355014801025, + "logits/rejected": -0.3321405053138733, + "logps/chosen": -162.58697509765625, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -331.218505859375, + "loss": 1.0617, + "margin_dpo/margin_mean": 140.24000549316406, + "margin_dpo/margin_std": 172.24667358398438, + "step": 207 + }, + { + "KL/chosen_KL_mean": -163.8634490966797, + "KL/mean": -219.51443481445312, + "KL/rejected_KL_mean": -275.1654052734375, + "KL/std": 141.97909545898438, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.0032390430569648743, + "fcm_dpo/delta": 0.06209877133369446, + "fcm_dpo/margin": 111.30195617675781, + "fcm_dpo/q_t": 0.42106446623802185, + "grad_norm": 23.454050064086914, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.34735146164894104, + "logits/rejected": -0.33190101385116577, + "logps/chosen": -230.93106079101562, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -369.4523010253906, + "loss": 1.1334, + "margin_dpo/margin_mean": 111.30195617675781, + "margin_dpo/margin_std": 172.6982879638672, + "step": 208 + }, + { + "KL/chosen_KL_mean": -133.59063720703125, + "KL/mean": -184.37820434570312, + "KL/rejected_KL_mean": -235.165771484375, + "KL/std": 125.87327575683594, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.0036668735556304455, + "fcm_dpo/delta": 0.042608536779880524, + "fcm_dpo/margin": 101.57514190673828, + "fcm_dpo/q_t": 0.41783520579338074, + "grad_norm": 21.281137466430664, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.257140576839447, + "logits/rejected": -0.24332758784294128, + "logps/chosen": -189.77232360839844, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -316.1072998046875, + "loss": 1.127, + "margin_dpo/margin_mean": 101.57514953613281, + "margin_dpo/margin_std": 156.05691528320312, + "step": 209 + }, + { + "KL/chosen_KL_mean": -117.53192138671875, + "KL/mean": -174.57296752929688, + "KL/rejected_KL_mean": -231.614013671875, + "KL/std": 124.3192367553711, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.0036693743895739317, + "fcm_dpo/delta": -0.03410163149237633, + "fcm_dpo/margin": 114.08209228515625, + "fcm_dpo/q_t": 0.4016813635826111, + "grad_norm": 30.14597511291504, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.3081471621990204, + "logits/rejected": -0.30119192600250244, + "logps/chosen": -163.90374755859375, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -308.2956237792969, + "loss": 1.0831, + "margin_dpo/margin_mean": 114.08209228515625, + "margin_dpo/margin_std": 149.70401000976562, + "step": 210 + }, + { + "KL/chosen_KL_mean": -176.67413330078125, + "KL/mean": -227.01296997070312, + "KL/rejected_KL_mean": -277.351806640625, + "KL/std": 148.22705078125, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.0037167323753237724, + "fcm_dpo/delta": 0.03998677805066109, + "fcm_dpo/margin": 100.67768859863281, + "fcm_dpo/q_t": 0.42228320240974426, + "grad_norm": 28.76727294921875, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.31403297185897827, + "logits/rejected": -0.27679428458213806, + "logps/chosen": -255.60647583007812, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -364.17279052734375, + "loss": 1.1458, + "margin_dpo/margin_mean": 100.67768859863281, + "margin_dpo/margin_std": 173.43338012695312, + "step": 211 + }, + { + "KL/chosen_KL_mean": -147.537109375, + "KL/mean": -220.67324829101562, + "KL/rejected_KL_mean": -293.80938720703125, + "KL/std": 159.14773559570312, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.0029512690380215645, + "fcm_dpo/delta": -0.09743355959653854, + "fcm_dpo/margin": 146.27227783203125, + "fcm_dpo/q_t": 0.3903607130050659, + "grad_norm": 18.54225730895996, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.26962530612945557, + "logits/rejected": -0.26270633935928345, + "logps/chosen": -205.73411560058594, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -396.86724853515625, + "loss": 1.0686, + "margin_dpo/margin_mean": 146.27227783203125, + "margin_dpo/margin_std": 174.09445190429688, + "step": 212 + }, + { + "KL/chosen_KL_mean": -143.3365478515625, + "KL/mean": -215.00723266601562, + "KL/rejected_KL_mean": -286.67791748046875, + "KL/std": 147.20899963378906, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.002868292387574911, + "fcm_dpo/delta": -0.02426821179687977, + "fcm_dpo/margin": 143.34136962890625, + "fcm_dpo/q_t": 0.4022514820098877, + "grad_norm": 18.237730026245117, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.2986302077770233, + "logits/rejected": -0.27630025148391724, + "logps/chosen": -210.84925842285156, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -380.5926513671875, + "loss": 1.0765, + "margin_dpo/margin_mean": 143.3413848876953, + "margin_dpo/margin_std": 173.92288208007812, + "step": 213 + }, + { + "KL/chosen_KL_mean": -125.64663696289062, + "KL/mean": -189.3560028076172, + "KL/rejected_KL_mean": -253.06533813476562, + "KL/std": 145.16001892089844, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.0030471929349005222, + "fcm_dpo/delta": 0.018730763345956802, + "fcm_dpo/margin": 127.41871643066406, + "fcm_dpo/q_t": 0.4126247763633728, + "grad_norm": 18.90703010559082, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.21006491780281067, + "logits/rejected": -0.2164653241634369, + "logps/chosen": -167.25152587890625, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -330.582763671875, + "loss": 1.0976, + "margin_dpo/margin_mean": 127.41871643066406, + "margin_dpo/margin_std": 170.29165649414062, + "step": 214 + }, + { + "KL/chosen_KL_mean": -150.6676025390625, + "KL/mean": -216.49374389648438, + "KL/rejected_KL_mean": -282.31988525390625, + "KL/std": 148.04367065429688, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.0030098496936261654, + "fcm_dpo/delta": 0.005100354552268982, + "fcm_dpo/margin": 131.6522979736328, + "fcm_dpo/q_t": 0.4081616997718811, + "grad_norm": 22.060007095336914, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.24693366885185242, + "logits/rejected": -0.23978078365325928, + "logps/chosen": -203.94686889648438, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -372.2845458984375, + "loss": 1.0802, + "margin_dpo/margin_mean": 131.6522979736328, + "margin_dpo/margin_std": 154.73556518554688, + "step": 215 + }, + { + "KL/chosen_KL_mean": -155.56214904785156, + "KL/mean": -220.8539581298828, + "KL/rejected_KL_mean": -286.145751953125, + "KL/std": 156.0887451171875, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.0031287854071706533, + "fcm_dpo/delta": -0.01758376508951187, + "fcm_dpo/margin": 130.5836181640625, + "fcm_dpo/q_t": 0.40592771768569946, + "grad_norm": 22.239622116088867, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.2485334426164627, + "logits/rejected": -0.23272496461868286, + "logps/chosen": -204.449951171875, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -363.3446960449219, + "loss": 1.1032, + "margin_dpo/margin_mean": 130.5836181640625, + "margin_dpo/margin_std": 189.82901000976562, + "step": 216 + }, + { + "KL/chosen_KL_mean": -158.0336456298828, + "KL/mean": -235.93881225585938, + "KL/rejected_KL_mean": -313.843994140625, + "KL/std": 162.12493896484375, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.0027228975668549538, + "fcm_dpo/delta": -0.045385316014289856, + "fcm_dpo/margin": 155.81036376953125, + "fcm_dpo/q_t": 0.39862823486328125, + "grad_norm": 20.71597671508789, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.25405189394950867, + "logits/rejected": -0.2514030337333679, + "logps/chosen": -207.8789520263672, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -413.92230224609375, + "loss": 1.0651, + "margin_dpo/margin_mean": 155.81036376953125, + "margin_dpo/margin_std": 185.34475708007812, + "step": 217 + }, + { + "KL/chosen_KL_mean": -167.77676391601562, + "KL/mean": -227.49313354492188, + "KL/rejected_KL_mean": -287.2095031738281, + "KL/std": 159.71775817871094, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.0029854662716388702, + "fcm_dpo/delta": 0.0675802081823349, + "fcm_dpo/margin": 119.43275451660156, + "fcm_dpo/q_t": 0.42418262362480164, + "grad_norm": 21.031944274902344, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.2911514639854431, + "logits/rejected": -0.28862863779067993, + "logps/chosen": -226.3534393310547, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -375.055908203125, + "loss": 1.1395, + "margin_dpo/margin_mean": 119.43275451660156, + "margin_dpo/margin_std": 189.45452880859375, + "step": 218 + }, + { + "KL/chosen_KL_mean": -179.43991088867188, + "KL/mean": -230.53575134277344, + "KL/rejected_KL_mean": -281.631591796875, + "KL/std": 166.12942504882812, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.003073344938457012, + "fcm_dpo/delta": -0.012668075039982796, + "fcm_dpo/margin": 102.19166564941406, + "fcm_dpo/q_t": 0.42826542258262634, + "grad_norm": 32.95735549926758, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.2939431965351105, + "logits/rejected": -0.29159048199653625, + "logps/chosen": -240.52377319335938, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -367.4620361328125, + "loss": 1.1924, + "margin_dpo/margin_mean": 102.19166564941406, + "margin_dpo/margin_std": 205.6397247314453, + "step": 219 + }, + { + "KL/chosen_KL_mean": -201.14923095703125, + "KL/mean": -243.14886474609375, + "KL/rejected_KL_mean": -285.1484680175781, + "KL/std": 140.5084228515625, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.003665705444291234, + "fcm_dpo/delta": 0.13320602476596832, + "fcm_dpo/margin": 83.9992446899414, + "fcm_dpo/q_t": 0.4367624521255493, + "grad_norm": 45.616695404052734, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.3121609687805176, + "logits/rejected": -0.2908840775489807, + "logps/chosen": -271.1805419921875, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -372.833984375, + "loss": 1.176, + "margin_dpo/margin_mean": 83.9992446899414, + "margin_dpo/margin_std": 149.43328857421875, + "step": 220 + }, + { + "KL/chosen_KL_mean": -161.646728515625, + "KL/mean": -256.31317138671875, + "KL/rejected_KL_mean": -350.9796142578125, + "KL/std": 178.03521728515625, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.0027057200204581022, + "fcm_dpo/delta": -0.25114211440086365, + "fcm_dpo/margin": 189.3328857421875, + "fcm_dpo/q_t": 0.35794326663017273, + "grad_norm": 40.98723602294922, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.3036612868309021, + "logits/rejected": -0.3129799962043762, + "logps/chosen": -213.80145263671875, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -457.44732666015625, + "loss": 1.0037, + "margin_dpo/margin_mean": 189.3328857421875, + "margin_dpo/margin_std": 184.73748779296875, + "step": 221 + }, + { + "KL/chosen_KL_mean": -166.11956787109375, + "KL/mean": -236.33322143554688, + "KL/rejected_KL_mean": -306.546875, + "KL/std": 162.46240234375, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.00264683086425066, + "fcm_dpo/delta": 0.03757670894265175, + "fcm_dpo/margin": 140.42733764648438, + "fcm_dpo/q_t": 0.4158090353012085, + "grad_norm": 24.47178840637207, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.3462597131729126, + "logits/rejected": -0.3537750244140625, + "logps/chosen": -227.09066772460938, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -406.54803466796875, + "loss": 1.1057, + "margin_dpo/margin_mean": 140.42733764648438, + "margin_dpo/margin_std": 183.39102172851562, + "step": 222 + }, + { + "KL/chosen_KL_mean": -183.58261108398438, + "KL/mean": -243.48226928710938, + "KL/rejected_KL_mean": -303.38189697265625, + "KL/std": 159.4056396484375, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.0028988937847316265, + "fcm_dpo/delta": 0.0806143581867218, + "fcm_dpo/margin": 119.79930114746094, + "fcm_dpo/q_t": 0.4264669120311737, + "grad_norm": 21.68163299560547, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.2645799219608307, + "logits/rejected": -0.2657017111778259, + "logps/chosen": -236.22317504882812, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -386.2069396972656, + "loss": 1.1509, + "margin_dpo/margin_mean": 119.79930114746094, + "margin_dpo/margin_std": 204.67282104492188, + "step": 223 + }, + { + "KL/chosen_KL_mean": -170.26473999023438, + "KL/mean": -239.272216796875, + "KL/rejected_KL_mean": -308.27972412109375, + "KL/std": 185.91464233398438, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.002813777420669794, + "fcm_dpo/delta": 0.0016366764903068542, + "fcm_dpo/margin": 138.0149383544922, + "fcm_dpo/q_t": 0.41247671842575073, + "grad_norm": 31.99540901184082, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.27732497453689575, + "logits/rejected": -0.26288285851478577, + "logps/chosen": -218.86013793945312, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -385.39617919921875, + "loss": 1.1282, + "margin_dpo/margin_mean": 138.0149383544922, + "margin_dpo/margin_std": 214.30078125, + "step": 224 + }, + { + "KL/chosen_KL_mean": -199.21405029296875, + "KL/mean": -281.2215576171875, + "KL/rejected_KL_mean": -363.2290954589844, + "KL/std": 178.91641235351562, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.0026704357005655766, + "fcm_dpo/delta": -0.07286377251148224, + "fcm_dpo/margin": 164.01502990722656, + "fcm_dpo/q_t": 0.3952982723712921, + "grad_norm": 19.550594329833984, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.24977388978004456, + "logits/rejected": -0.24200385808944702, + "logps/chosen": -257.21453857421875, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -463.13201904296875, + "loss": 1.0792, + "margin_dpo/margin_mean": 164.01502990722656, + "margin_dpo/margin_std": 227.45672607421875, + "step": 225 + }, + { + "KL/chosen_KL_mean": -171.4937286376953, + "KL/mean": -237.76144409179688, + "KL/rejected_KL_mean": -304.0291748046875, + "KL/std": 178.0089111328125, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.0027313155587762594, + "fcm_dpo/delta": 0.056763019412755966, + "fcm_dpo/margin": 132.5354461669922, + "fcm_dpo/q_t": 0.42256051301956177, + "grad_norm": 24.24056053161621, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.32699283957481384, + "logits/rejected": -0.29330500960350037, + "logps/chosen": -230.39251708984375, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -382.7169189453125, + "loss": 1.139, + "margin_dpo/margin_mean": 132.53546142578125, + "margin_dpo/margin_std": 209.61676025390625, + "step": 226 + }, + { + "KL/chosen_KL_mean": -197.4924774169922, + "KL/mean": -280.9031677246094, + "KL/rejected_KL_mean": -364.3138427734375, + "KL/std": 201.27569580078125, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.002555706538259983, + "fcm_dpo/delta": -0.05992633104324341, + "fcm_dpo/margin": 166.82138061523438, + "fcm_dpo/q_t": 0.3967394530773163, + "grad_norm": 19.361919403076172, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.2924739718437195, + "logits/rejected": -0.2799687385559082, + "logps/chosen": -256.56463623046875, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -463.7261962890625, + "loss": 1.072, + "margin_dpo/margin_mean": 166.82138061523438, + "margin_dpo/margin_std": 210.5462646484375, + "step": 227 + }, + { + "KL/chosen_KL_mean": -204.37904357910156, + "KL/mean": -260.72320556640625, + "KL/rejected_KL_mean": -317.0673828125, + "KL/std": 166.0177001953125, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.0030394818168133497, + "fcm_dpo/delta": 0.08101288229227066, + "fcm_dpo/margin": 112.68833923339844, + "fcm_dpo/q_t": 0.427123486995697, + "grad_norm": 26.379526138305664, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.28412914276123047, + "logits/rejected": -0.26933881640434265, + "logps/chosen": -270.27032470703125, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -408.1161193847656, + "loss": 1.1678, + "margin_dpo/margin_mean": 112.68833923339844, + "margin_dpo/margin_std": 209.85467529296875, + "step": 228 + }, + { + "KL/chosen_KL_mean": -214.5729522705078, + "KL/mean": -271.60009765625, + "KL/rejected_KL_mean": -328.62725830078125, + "KL/std": 190.02706909179688, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.0032839591149240732, + "fcm_dpo/delta": 0.04005454480648041, + "fcm_dpo/margin": 114.0543212890625, + "fcm_dpo/q_t": 0.4181785583496094, + "grad_norm": 38.73786544799805, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.3614084720611572, + "logits/rejected": -0.3334274888038635, + "logps/chosen": -285.2793273925781, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -413.1546630859375, + "loss": 1.1432, + "margin_dpo/margin_mean": 114.0543212890625, + "margin_dpo/margin_std": 198.13516235351562, + "step": 229 + }, + { + "KL/chosen_KL_mean": -155.9505157470703, + "KL/mean": -249.08929443359375, + "KL/rejected_KL_mean": -342.22808837890625, + "KL/std": 169.89035034179688, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.0025410507805645466, + "fcm_dpo/delta": -0.15831325948238373, + "fcm_dpo/margin": 186.27757263183594, + "fcm_dpo/q_t": 0.3732534348964691, + "grad_norm": 28.544626235961914, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.25178319215774536, + "logits/rejected": -0.25868701934814453, + "logps/chosen": -195.23251342773438, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -427.8499755859375, + "loss": 1.0197, + "margin_dpo/margin_mean": 186.27755737304688, + "margin_dpo/margin_std": 184.69554138183594, + "step": 230 + }, + { + "KL/chosen_KL_mean": -179.1609344482422, + "KL/mean": -240.4979705810547, + "KL/rejected_KL_mean": -301.8349914550781, + "KL/std": 151.0618896484375, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.0027899318374693394, + "fcm_dpo/delta": 0.0851934552192688, + "fcm_dpo/margin": 122.67406463623047, + "fcm_dpo/q_t": 0.427597314119339, + "grad_norm": 24.108116149902344, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.3525630831718445, + "logits/rejected": -0.32520967721939087, + "logps/chosen": -242.4373779296875, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -375.9588928222656, + "loss": 1.1322, + "margin_dpo/margin_mean": 122.67406463623047, + "margin_dpo/margin_std": 179.02493286132812, + "step": 231 + }, + { + "KL/chosen_KL_mean": -216.9395294189453, + "KL/mean": -265.2451171875, + "KL/rejected_KL_mean": -313.55072021484375, + "KL/std": 175.61700439453125, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.0033727658446878195, + "fcm_dpo/delta": 0.10968147218227386, + "fcm_dpo/margin": 96.61114501953125, + "fcm_dpo/q_t": 0.435283899307251, + "grad_norm": 27.411245346069336, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.3760000467300415, + "logits/rejected": -0.35748612880706787, + "logps/chosen": -287.68829345703125, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -397.52777099609375, + "loss": 1.1718, + "margin_dpo/margin_mean": 96.61115264892578, + "margin_dpo/margin_std": 181.7001495361328, + "step": 232 + }, + { + "KL/chosen_KL_mean": -181.86721801757812, + "KL/mean": -262.99560546875, + "KL/rejected_KL_mean": -344.12396240234375, + "KL/std": 183.73455810546875, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.0029199281707406044, + "fcm_dpo/delta": -0.14302438497543335, + "fcm_dpo/margin": 162.25674438476562, + "fcm_dpo/q_t": 0.38662809133529663, + "grad_norm": 21.30043601989746, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.3732132017612457, + "logits/rejected": -0.37482750415802, + "logps/chosen": -236.75015258789062, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -451.6040344238281, + "loss": 1.0754, + "margin_dpo/margin_mean": 162.25674438476562, + "margin_dpo/margin_std": 236.0518035888672, + "step": 233 + }, + { + "KL/chosen_KL_mean": -163.74453735351562, + "KL/mean": -245.33642578125, + "KL/rejected_KL_mean": -326.9283447265625, + "KL/std": 157.0659942626953, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.0024456209503114223, + "fcm_dpo/delta": -0.015596166253089905, + "fcm_dpo/margin": 163.18380737304688, + "fcm_dpo/q_t": 0.403796911239624, + "grad_norm": 22.082944869995117, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.36095279455184937, + "logits/rejected": -0.3873627483844757, + "logps/chosen": -207.83905029296875, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -426.93499755859375, + "loss": 1.0777, + "margin_dpo/margin_mean": 163.18380737304688, + "margin_dpo/margin_std": 176.453857421875, + "step": 234 + }, + { + "KL/chosen_KL_mean": -201.49075317382812, + "KL/mean": -251.1799774169922, + "KL/rejected_KL_mean": -300.86920166015625, + "KL/std": 150.57394409179688, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.002739574294537306, + "fcm_dpo/delta": 0.046936918050050735, + "fcm_dpo/margin": 99.37845611572266, + "fcm_dpo/q_t": 0.43904921412467957, + "grad_norm": 21.031633377075195, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.38550442457199097, + "logits/rejected": -0.3654111921787262, + "logps/chosen": -263.7286682128906, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -391.2642822265625, + "loss": 1.1915, + "margin_dpo/margin_mean": 99.37844848632812, + "margin_dpo/margin_std": 185.09054565429688, + "step": 235 + }, + { + "KL/chosen_KL_mean": -145.79144287109375, + "KL/mean": -228.65101623535156, + "KL/rejected_KL_mean": -311.5105895996094, + "KL/std": 160.1380157470703, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.0026162322610616684, + "fcm_dpo/delta": -0.06015552580356598, + "fcm_dpo/margin": 165.71914672851562, + "fcm_dpo/q_t": 0.3923841118812561, + "grad_norm": 37.16407775878906, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.3568821847438812, + "logits/rejected": -0.3578595221042633, + "logps/chosen": -195.1328125, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -415.022216796875, + "loss": 1.0388, + "margin_dpo/margin_mean": 165.71914672851562, + "margin_dpo/margin_std": 155.69921875, + "step": 236 + }, + { + "KL/chosen_KL_mean": -203.95053100585938, + "KL/mean": -268.9967041015625, + "KL/rejected_KL_mean": -334.04290771484375, + "KL/std": 166.29592895507812, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0028002336621284485, + "fcm_dpo/delta": 0.05379777029156685, + "fcm_dpo/margin": 130.09239196777344, + "fcm_dpo/q_t": 0.4198486804962158, + "grad_norm": 27.22985076904297, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.32136499881744385, + "logits/rejected": -0.3153060972690582, + "logps/chosen": -258.11865234375, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -428.82330322265625, + "loss": 1.1202, + "margin_dpo/margin_mean": 130.09239196777344, + "margin_dpo/margin_std": 190.1111602783203, + "step": 237 + }, + { + "KL/chosen_KL_mean": -188.8069610595703, + "KL/mean": -254.78656005859375, + "KL/rejected_KL_mean": -320.7662048339844, + "KL/std": 168.50390625, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0028396020643413067, + "fcm_dpo/delta": 0.035806380212306976, + "fcm_dpo/margin": 131.95924377441406, + "fcm_dpo/q_t": 0.418282687664032, + "grad_norm": 23.320514678955078, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.31921494007110596, + "logits/rejected": -0.33235523104667664, + "logps/chosen": -242.78009033203125, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -410.18414306640625, + "loss": 1.116, + "margin_dpo/margin_mean": 131.95924377441406, + "margin_dpo/margin_std": 188.6099395751953, + "step": 238 + }, + { + "KL/chosen_KL_mean": -209.04046630859375, + "KL/mean": -283.45135498046875, + "KL/rejected_KL_mean": -357.8622741699219, + "KL/std": 165.2271728515625, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.0027087791822850704, + "fcm_dpo/delta": -0.025651507079601288, + "fcm_dpo/margin": 148.8218231201172, + "fcm_dpo/q_t": 0.4025055170059204, + "grad_norm": 26.025190353393555, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.3110630214214325, + "logits/rejected": -0.3010329604148865, + "logps/chosen": -267.1383056640625, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -451.4552307128906, + "loss": 1.0886, + "margin_dpo/margin_mean": 148.8218231201172, + "margin_dpo/margin_std": 182.97344970703125, + "step": 239 + }, + { + "KL/chosen_KL_mean": -216.27822875976562, + "KL/mean": -275.22637939453125, + "KL/rejected_KL_mean": -334.17449951171875, + "KL/std": 174.17791748046875, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.003052940359339118, + "fcm_dpo/delta": 0.06172945722937584, + "fcm_dpo/margin": 117.89629364013672, + "fcm_dpo/q_t": 0.42239585518836975, + "grad_norm": 30.3389949798584, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.34589529037475586, + "logits/rejected": -0.32597455382347107, + "logps/chosen": -276.8927001953125, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -408.29302978515625, + "loss": 1.1595, + "margin_dpo/margin_mean": 117.89629364013672, + "margin_dpo/margin_std": 205.7227020263672, + "step": 240 + }, + { + "KL/chosen_KL_mean": -189.63717651367188, + "KL/mean": -283.8388366699219, + "KL/rejected_KL_mean": -378.04052734375, + "KL/std": 193.572021484375, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.002562090754508972, + "fcm_dpo/delta": -0.16558168828487396, + "fcm_dpo/margin": 188.4033203125, + "fcm_dpo/q_t": 0.3756968379020691, + "grad_norm": 19.404237747192383, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.3840920329093933, + "logits/rejected": -0.36620739102363586, + "logps/chosen": -255.72824096679688, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -466.1014099121094, + "loss": 1.033, + "margin_dpo/margin_mean": 188.4033203125, + "margin_dpo/margin_std": 218.57650756835938, + "step": 241 + }, + { + "KL/chosen_KL_mean": -209.334716796875, + "KL/mean": -275.63055419921875, + "KL/rejected_KL_mean": -341.9263610839844, + "KL/std": 168.1392364501953, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.0026945569552481174, + "fcm_dpo/delta": 0.052527863532304764, + "fcm_dpo/margin": 132.59165954589844, + "fcm_dpo/q_t": 0.4214203357696533, + "grad_norm": 27.632558822631836, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.3013432025909424, + "logits/rejected": -0.27941906452178955, + "logps/chosen": -277.1986389160156, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -425.2867126464844, + "loss": 1.1318, + "margin_dpo/margin_mean": 132.59165954589844, + "margin_dpo/margin_std": 201.74139404296875, + "step": 242 + }, + { + "KL/chosen_KL_mean": -208.54391479492188, + "KL/mean": -289.72686767578125, + "KL/rejected_KL_mean": -370.9097900390625, + "KL/std": 181.60488891601562, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.0025174664333462715, + "fcm_dpo/delta": -0.015545245260000229, + "fcm_dpo/margin": 162.36587524414062, + "fcm_dpo/q_t": 0.4042058289051056, + "grad_norm": 18.131187438964844, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.2907285690307617, + "logits/rejected": -0.26282238960266113, + "logps/chosen": -271.628173828125, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -447.24542236328125, + "loss": 1.0785, + "margin_dpo/margin_mean": 162.36585998535156, + "margin_dpo/margin_std": 201.73919677734375, + "step": 243 + }, + { + "KL/chosen_KL_mean": -176.4449462890625, + "KL/mean": -262.936767578125, + "KL/rejected_KL_mean": -349.4285888671875, + "KL/std": 177.73191833496094, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.002386314095929265, + "fcm_dpo/delta": -0.024043917655944824, + "fcm_dpo/margin": 172.983642578125, + "fcm_dpo/q_t": 0.4017675518989563, + "grad_norm": 22.16912841796875, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.37594205141067505, + "logits/rejected": -0.3709765076637268, + "logps/chosen": -237.58563232421875, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -444.3205261230469, + "loss": 1.061, + "margin_dpo/margin_mean": 172.983642578125, + "margin_dpo/margin_std": 182.35865783691406, + "step": 244 + }, + { + "KL/chosen_KL_mean": -210.67608642578125, + "KL/mean": -275.7692565917969, + "KL/rejected_KL_mean": -340.8624267578125, + "KL/std": 175.44842529296875, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.002276795916259289, + "fcm_dpo/delta": -0.028318271040916443, + "fcm_dpo/margin": 130.18634033203125, + "fcm_dpo/q_t": 0.42754846811294556, + "grad_norm": 21.60332489013672, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.38238322734832764, + "logits/rejected": -0.35831815004348755, + "logps/chosen": -277.9383544921875, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -428.50250244140625, + "loss": 1.1617, + "margin_dpo/margin_mean": 130.1863250732422, + "margin_dpo/margin_std": 192.90383911132812, + "step": 245 + }, + { + "KL/chosen_KL_mean": -203.3482666015625, + "KL/mean": -271.3511657714844, + "KL/rejected_KL_mean": -339.3540344238281, + "KL/std": 193.9945068359375, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.002540751826018095, + "fcm_dpo/delta": 0.08369514346122742, + "fcm_dpo/margin": 136.00575256347656, + "fcm_dpo/q_t": 0.42736515402793884, + "grad_norm": 29.36928939819336, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.43162864446640015, + "logits/rejected": -0.39635908603668213, + "logps/chosen": -270.04522705078125, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -423.70037841796875, + "loss": 1.139, + "margin_dpo/margin_mean": 136.00576782226562, + "margin_dpo/margin_std": 211.0897216796875, + "step": 246 + }, + { + "KL/chosen_KL_mean": -238.23548889160156, + "KL/mean": -336.6441345214844, + "KL/rejected_KL_mean": -435.05279541015625, + "KL/std": 201.9400634765625, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.002273306716233492, + "fcm_dpo/delta": -0.09434907883405685, + "fcm_dpo/margin": 196.8173065185547, + "fcm_dpo/q_t": 0.3884933590888977, + "grad_norm": 24.273239135742188, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.35952645540237427, + "logits/rejected": -0.35816654562950134, + "logps/chosen": -294.8408203125, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -541.3460693359375, + "loss": 1.0487, + "margin_dpo/margin_mean": 196.81732177734375, + "margin_dpo/margin_std": 221.4100799560547, + "step": 247 + }, + { + "KL/chosen_KL_mean": -203.11044311523438, + "KL/mean": -287.45843505859375, + "KL/rejected_KL_mean": -371.806396484375, + "KL/std": 163.28530883789062, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.002244535367935896, + "fcm_dpo/delta": 0.031689293682575226, + "fcm_dpo/margin": 168.69598388671875, + "fcm_dpo/q_t": 0.4118279814720154, + "grad_norm": 20.322803497314453, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.37604576349258423, + "logits/rejected": -0.39418306946754456, + "logps/chosen": -247.15365600585938, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -463.66326904296875, + "loss": 1.0792, + "margin_dpo/margin_mean": 168.69598388671875, + "margin_dpo/margin_std": 161.9207763671875, + "step": 248 + }, + { + "KL/chosen_KL_mean": -261.7310791015625, + "KL/mean": -307.686279296875, + "KL/rejected_KL_mean": -353.6415100097656, + "KL/std": 179.0096435546875, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.002784975804388523, + "fcm_dpo/delta": 0.0796995759010315, + "fcm_dpo/margin": 91.9104232788086, + "fcm_dpo/q_t": 0.4464285671710968, + "grad_norm": 43.7970085144043, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.40871942043304443, + "logits/rejected": -0.38268932700157166, + "logps/chosen": -324.1734313964844, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -434.10955810546875, + "loss": 1.2311, + "margin_dpo/margin_mean": 91.9104232788086, + "margin_dpo/margin_std": 214.4239959716797, + "step": 249 + }, + { + "KL/chosen_KL_mean": -232.26882934570312, + "KL/mean": -313.94488525390625, + "KL/rejected_KL_mean": -395.6209716796875, + "KL/std": 176.32708740234375, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.002672810573130846, + "fcm_dpo/delta": -0.06773751974105835, + "fcm_dpo/margin": 163.3521270751953, + "fcm_dpo/q_t": 0.3925377130508423, + "grad_norm": 46.0150032043457, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.4073798656463623, + "logits/rejected": -0.3506365418434143, + "logps/chosen": -297.905517578125, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -469.4928283691406, + "loss": 1.059, + "margin_dpo/margin_mean": 163.3521270751953, + "margin_dpo/margin_std": 180.181884765625, + "step": 250 + }, + { + "KL/chosen_KL_mean": -246.01177978515625, + "KL/mean": -302.828857421875, + "KL/rejected_KL_mean": -359.64593505859375, + "KL/std": 187.79159545898438, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.0028411387465894222, + "fcm_dpo/delta": 0.11464589834213257, + "fcm_dpo/margin": 113.63416290283203, + "fcm_dpo/q_t": 0.43381422758102417, + "grad_norm": 41.136512756347656, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.37190985679626465, + "logits/rejected": -0.3395519554615021, + "logps/chosen": -303.1944885253906, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -437.30938720703125, + "loss": 1.1687, + "margin_dpo/margin_mean": 113.63416290283203, + "margin_dpo/margin_std": 201.274169921875, + "step": 251 + }, + { + "KL/chosen_KL_mean": -241.49786376953125, + "KL/mean": -317.1058654785156, + "KL/rejected_KL_mean": -392.7138671875, + "KL/std": 162.52325439453125, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.0027777920477092266, + "fcm_dpo/delta": -0.0407252199947834, + "fcm_dpo/margin": 151.21600341796875, + "fcm_dpo/q_t": 0.3957163393497467, + "grad_norm": 25.602100372314453, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.42717283964157104, + "logits/rejected": -0.38557177782058716, + "logps/chosen": -313.1835021972656, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -477.47186279296875, + "loss": 1.0596, + "margin_dpo/margin_mean": 151.21600341796875, + "margin_dpo/margin_std": 162.9154815673828, + "step": 252 + }, + { + "KL/chosen_KL_mean": -202.2740478515625, + "KL/mean": -278.83538818359375, + "KL/rejected_KL_mean": -355.396728515625, + "KL/std": 169.63818359375, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0027958948630839586, + "fcm_dpo/delta": -0.05647484213113785, + "fcm_dpo/margin": 153.12266540527344, + "fcm_dpo/q_t": 0.3956605792045593, + "grad_norm": 26.383848190307617, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.42702776193618774, + "logits/rejected": -0.4189242720603943, + "logps/chosen": -271.4079895019531, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -454.0992736816406, + "loss": 1.0808, + "margin_dpo/margin_mean": 153.1226806640625, + "margin_dpo/margin_std": 188.74114990234375, + "step": 253 + }, + { + "KL/chosen_KL_mean": -190.6053924560547, + "KL/mean": -255.74807739257812, + "KL/rejected_KL_mean": -320.8907470703125, + "KL/std": 174.94302368164062, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.002762971445918083, + "fcm_dpo/delta": 0.06207642704248428, + "fcm_dpo/margin": 130.28536987304688, + "fcm_dpo/q_t": 0.4237990379333496, + "grad_norm": 26.932371139526367, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.40052559971809387, + "logits/rejected": -0.38795092701911926, + "logps/chosen": -244.76039123535156, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -411.1983947753906, + "loss": 1.1403, + "margin_dpo/margin_mean": 130.28536987304688, + "margin_dpo/margin_std": 217.71649169921875, + "step": 254 + }, + { + "KL/chosen_KL_mean": -196.3863983154297, + "KL/mean": -262.91351318359375, + "KL/rejected_KL_mean": -329.44061279296875, + "KL/std": 153.0009002685547, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.002854404039680958, + "fcm_dpo/delta": 0.02931365743279457, + "fcm_dpo/margin": 133.05422973632812, + "fcm_dpo/q_t": 0.41438183188438416, + "grad_norm": 25.421916961669922, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.35080671310424805, + "logits/rejected": -0.3392289876937866, + "logps/chosen": -253.528076171875, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -419.64923095703125, + "loss": 1.1183, + "margin_dpo/margin_mean": 133.05422973632812, + "margin_dpo/margin_std": 194.57473754882812, + "step": 255 + }, + { + "KL/chosen_KL_mean": -169.05984497070312, + "KL/mean": -243.46347045898438, + "KL/rejected_KL_mean": -317.8670959472656, + "KL/std": 165.87603759765625, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.002850246848538518, + "fcm_dpo/delta": -0.04200413078069687, + "fcm_dpo/margin": 148.8072509765625, + "fcm_dpo/q_t": 0.39909690618515015, + "grad_norm": 28.732080459594727, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.2951539158821106, + "logits/rejected": -0.3040773868560791, + "logps/chosen": -224.22332763671875, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -410.42999267578125, + "loss": 1.075, + "margin_dpo/margin_mean": 148.8072509765625, + "margin_dpo/margin_std": 190.7834014892578, + "step": 256 + }, + { + "KL/chosen_KL_mean": -165.35116577148438, + "KL/mean": -242.6610107421875, + "KL/rejected_KL_mean": -319.9708557128906, + "KL/std": 172.49417114257812, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.002569081261754036, + "fcm_dpo/delta": -0.008627064526081085, + "fcm_dpo/margin": 154.6196746826172, + "fcm_dpo/q_t": 0.40691322088241577, + "grad_norm": 21.039451599121094, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.33848869800567627, + "logits/rejected": -0.3249668478965759, + "logps/chosen": -214.77487182617188, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -399.5087890625, + "loss": 1.0937, + "margin_dpo/margin_mean": 154.6196746826172, + "margin_dpo/margin_std": 196.26806640625, + "step": 257 + }, + { + "KL/chosen_KL_mean": -220.045654296875, + "KL/mean": -296.9225158691406, + "KL/rejected_KL_mean": -373.79937744140625, + "KL/std": 184.12759399414062, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.0026925005950033665, + "fcm_dpo/delta": -0.024290982633829117, + "fcm_dpo/margin": 153.7537078857422, + "fcm_dpo/q_t": 0.4024388790130615, + "grad_norm": 26.345495223999023, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.3109281659126282, + "logits/rejected": -0.3115309476852417, + "logps/chosen": -279.4297790527344, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -469.78948974609375, + "loss": 1.0982, + "margin_dpo/margin_mean": 153.7537078857422, + "margin_dpo/margin_std": 222.58351135253906, + "step": 258 + }, + { + "KL/chosen_KL_mean": -209.99713134765625, + "KL/mean": -273.5628356933594, + "KL/rejected_KL_mean": -337.1285400390625, + "KL/std": 172.61416625976562, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.0028006834909319878, + "fcm_dpo/delta": 0.06836118549108505, + "fcm_dpo/margin": 127.13138580322266, + "fcm_dpo/q_t": 0.42346620559692383, + "grad_norm": 21.578977584838867, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.3506283462047577, + "logits/rejected": -0.3485710322856903, + "logps/chosen": -262.82550048828125, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -426.3201904296875, + "loss": 1.137, + "margin_dpo/margin_mean": 127.13138580322266, + "margin_dpo/margin_std": 197.51239013671875, + "step": 259 + }, + { + "KL/chosen_KL_mean": -210.90850830078125, + "KL/mean": -298.40771484375, + "KL/rejected_KL_mean": -385.90692138671875, + "KL/std": 181.40826416015625, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.002633176278322935, + "fcm_dpo/delta": -0.11144484579563141, + "fcm_dpo/margin": 174.99844360351562, + "fcm_dpo/q_t": 0.3861405551433563, + "grad_norm": 28.88026237487793, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.31534501910209656, + "logits/rejected": -0.33186644315719604, + "logps/chosen": -258.326171875, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -480.9967041015625, + "loss": 1.0402, + "margin_dpo/margin_mean": 174.99844360351562, + "margin_dpo/margin_std": 197.2706298828125, + "step": 260 + }, + { + "KL/chosen_KL_mean": -220.949462890625, + "KL/mean": -301.2371826171875, + "KL/rejected_KL_mean": -381.52496337890625, + "KL/std": 196.11915588378906, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0024081161245703697, + "fcm_dpo/delta": 0.020973602309823036, + "fcm_dpo/margin": 160.57550048828125, + "fcm_dpo/q_t": 0.41286349296569824, + "grad_norm": 18.689228057861328, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.2601136565208435, + "logits/rejected": -0.2584311366081238, + "logps/chosen": -273.9808349609375, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -470.0399169921875, + "loss": 1.1028, + "margin_dpo/margin_mean": 160.57550048828125, + "margin_dpo/margin_std": 221.1414337158203, + "step": 261 + }, + { + "KL/chosen_KL_mean": -260.35552978515625, + "KL/mean": -325.7707214355469, + "KL/rejected_KL_mean": -391.1859130859375, + "KL/std": 179.41098022460938, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.002791064791381359, + "fcm_dpo/delta": 0.04618440195918083, + "fcm_dpo/margin": 130.8303680419922, + "fcm_dpo/q_t": 0.4176996350288391, + "grad_norm": 33.41985321044922, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.29340440034866333, + "logits/rejected": -0.26309913396835327, + "logps/chosen": -319.9756774902344, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -477.60443115234375, + "loss": 1.1146, + "margin_dpo/margin_mean": 130.83035278320312, + "margin_dpo/margin_std": 180.4952392578125, + "step": 262 + }, + { + "KL/chosen_KL_mean": -233.8137969970703, + "KL/mean": -327.53448486328125, + "KL/rejected_KL_mean": -421.2552185058594, + "KL/std": 211.76744079589844, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.0023032082244753838, + "fcm_dpo/delta": -0.07411213964223862, + "fcm_dpo/margin": 187.44140625, + "fcm_dpo/q_t": 0.39327767491340637, + "grad_norm": 18.634008407592773, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.3248862028121948, + "logits/rejected": -0.30047309398651123, + "logps/chosen": -293.2347412109375, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -518.1124267578125, + "loss": 1.0674, + "margin_dpo/margin_mean": 187.44142150878906, + "margin_dpo/margin_std": 231.066162109375, + "step": 263 + }, + { + "KL/chosen_KL_mean": -241.53146362304688, + "KL/mean": -323.41162109375, + "KL/rejected_KL_mean": -405.291748046875, + "KL/std": 194.24461364746094, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.002298943232744932, + "fcm_dpo/delta": 0.028040196746587753, + "fcm_dpo/margin": 163.76028442382812, + "fcm_dpo/q_t": 0.4143482446670532, + "grad_norm": 20.764347076416016, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.3569568693637848, + "logits/rejected": -0.34849122166633606, + "logps/chosen": -304.2535400390625, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -499.14794921875, + "loss": 1.1174, + "margin_dpo/margin_mean": 163.76028442382812, + "margin_dpo/margin_std": 224.1405487060547, + "step": 264 + }, + { + "KL/chosen_KL_mean": -264.88433837890625, + "KL/mean": -340.1336669921875, + "KL/rejected_KL_mean": -415.3829650878906, + "KL/std": 215.2719268798828, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.0025752345100045204, + "fcm_dpo/delta": 0.020102664828300476, + "fcm_dpo/margin": 150.49859619140625, + "fcm_dpo/q_t": 0.4153306484222412, + "grad_norm": 23.874309539794922, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.31058114767074585, + "logits/rejected": -0.2928779423236847, + "logps/chosen": -326.8558349609375, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -503.403564453125, + "loss": 1.1349, + "margin_dpo/margin_mean": 150.49859619140625, + "margin_dpo/margin_std": 257.16748046875, + "step": 265 + }, + { + "KL/chosen_KL_mean": -264.1705322265625, + "KL/mean": -323.71405029296875, + "KL/rejected_KL_mean": -383.25750732421875, + "KL/std": 179.75607299804688, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0029307270888239145, + "fcm_dpo/delta": 0.07655003666877747, + "fcm_dpo/margin": 119.08696746826172, + "fcm_dpo/q_t": 0.42644163966178894, + "grad_norm": 46.22469711303711, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.3636462092399597, + "logits/rejected": -0.3230510354042053, + "logps/chosen": -331.27020263671875, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -451.228759765625, + "loss": 1.1556, + "margin_dpo/margin_mean": 119.08696746826172, + "margin_dpo/margin_std": 211.2481689453125, + "step": 266 + }, + { + "KL/chosen_KL_mean": -236.5076904296875, + "KL/mean": -311.24676513671875, + "KL/rejected_KL_mean": -385.98590087890625, + "KL/std": 193.96978759765625, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.00275492575019598, + "fcm_dpo/delta": -0.029174409806728363, + "fcm_dpo/margin": 149.47821044921875, + "fcm_dpo/q_t": 0.4047339856624603, + "grad_norm": 41.7883415222168, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.3469699025154114, + "logits/rejected": -0.3102799355983734, + "logps/chosen": -305.47845458984375, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -476.1543273925781, + "loss": 1.1054, + "margin_dpo/margin_mean": 149.47821044921875, + "margin_dpo/margin_std": 224.31707763671875, + "step": 267 + }, + { + "KL/chosen_KL_mean": -240.66119384765625, + "KL/mean": -317.0205078125, + "KL/rejected_KL_mean": -393.37982177734375, + "KL/std": 185.6231689453125, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.002668085042387247, + "fcm_dpo/delta": -0.01804983615875244, + "fcm_dpo/margin": 152.71859741210938, + "fcm_dpo/q_t": 0.40581750869750977, + "grad_norm": 28.408159255981445, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.35881197452545166, + "logits/rejected": -0.36506927013397217, + "logps/chosen": -296.5615234375, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -495.0274353027344, + "loss": 1.1018, + "margin_dpo/margin_mean": 152.71859741210938, + "margin_dpo/margin_std": 222.39694213867188, + "step": 268 + }, + { + "KL/chosen_KL_mean": -257.70648193359375, + "KL/mean": -346.0221862792969, + "KL/rejected_KL_mean": -434.337890625, + "KL/std": 184.19381713867188, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.00247764540836215, + "fcm_dpo/delta": -0.06811105459928513, + "fcm_dpo/margin": 176.63140869140625, + "fcm_dpo/q_t": 0.3938090205192566, + "grad_norm": 22.41498374938965, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.38506025075912476, + "logits/rejected": -0.3642171025276184, + "logps/chosen": -327.74603271484375, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -541.687255859375, + "loss": 1.0757, + "margin_dpo/margin_mean": 176.6313934326172, + "margin_dpo/margin_std": 235.69393920898438, + "step": 269 + }, + { + "KL/chosen_KL_mean": -222.79367065429688, + "KL/mean": -289.4638671875, + "KL/rejected_KL_mean": -356.134033203125, + "KL/std": 165.93325805664062, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.0027032734360545874, + "fcm_dpo/delta": 0.05463853478431702, + "fcm_dpo/margin": 133.3403778076172, + "fcm_dpo/q_t": 0.4207903742790222, + "grad_norm": 25.0926570892334, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.43856489658355713, + "logits/rejected": -0.4321529269218445, + "logps/chosen": -292.3271484375, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -466.06268310546875, + "loss": 1.1388, + "margin_dpo/margin_mean": 133.3403778076172, + "margin_dpo/margin_std": 216.5740509033203, + "step": 270 + }, + { + "KL/chosen_KL_mean": -203.2021942138672, + "KL/mean": -287.8795166015625, + "KL/rejected_KL_mean": -372.5568542480469, + "KL/std": 168.32081604003906, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.0024533187970519066, + "fcm_dpo/delta": -0.03273054212331772, + "fcm_dpo/margin": 169.3546905517578, + "fcm_dpo/q_t": 0.4006693959236145, + "grad_norm": 23.280786514282227, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.3510928452014923, + "logits/rejected": -0.33294203877449036, + "logps/chosen": -259.9667663574219, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -465.0706787109375, + "loss": 1.0663, + "margin_dpo/margin_mean": 169.3546905517578, + "margin_dpo/margin_std": 192.665283203125, + "step": 271 + }, + { + "KL/chosen_KL_mean": -191.9091339111328, + "KL/mean": -292.7960205078125, + "KL/rejected_KL_mean": -393.68292236328125, + "KL/std": 186.9519805908203, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.002250020857900381, + "fcm_dpo/delta": -0.09791300445795059, + "fcm_dpo/margin": 201.77377319335938, + "fcm_dpo/q_t": 0.38448095321655273, + "grad_norm": 28.866960525512695, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.29558974504470825, + "logits/rejected": -0.3049769401550293, + "logps/chosen": -241.4062957763672, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -499.2257080078125, + "loss": 1.0321, + "margin_dpo/margin_mean": 201.7737579345703, + "margin_dpo/margin_std": 194.00624084472656, + "step": 272 + }, + { + "KL/chosen_KL_mean": -226.32107543945312, + "KL/mean": -325.10498046875, + "KL/rejected_KL_mean": -423.8888244628906, + "KL/std": 189.16091918945312, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.0020193979144096375, + "fcm_dpo/delta": 0.0005125030875205994, + "fcm_dpo/margin": 197.56777954101562, + "fcm_dpo/q_t": 0.4059392511844635, + "grad_norm": 17.918615341186523, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.31307101249694824, + "logits/rejected": -0.29330283403396606, + "logps/chosen": -289.2964782714844, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -516.3873901367188, + "loss": 1.0715, + "margin_dpo/margin_mean": 197.56777954101562, + "margin_dpo/margin_std": 214.41448974609375, + "step": 273 + }, + { + "KL/chosen_KL_mean": -267.08245849609375, + "KL/mean": -342.52392578125, + "KL/rejected_KL_mean": -417.9654541015625, + "KL/std": 174.84378051757812, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.002351115457713604, + "fcm_dpo/delta": 0.06252269446849823, + "fcm_dpo/margin": 150.8830108642578, + "fcm_dpo/q_t": 0.420854389667511, + "grad_norm": 22.64203453063965, + "learning_rate": 3.75e-07, + "logits/chosen": -0.23737865686416626, + "logits/rejected": -0.22025717794895172, + "logps/chosen": -322.7501525878906, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -495.29852294921875, + "loss": 1.1162, + "margin_dpo/margin_mean": 150.88299560546875, + "margin_dpo/margin_std": 200.55410766601562, + "step": 274 + }, + { + "KL/chosen_KL_mean": -211.66796875, + "KL/mean": -292.65130615234375, + "KL/rejected_KL_mean": -373.6346435546875, + "KL/std": 179.12078857421875, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.002439269330352545, + "fcm_dpo/delta": 0.004620321094989777, + "fcm_dpo/margin": 161.9666748046875, + "fcm_dpo/q_t": 0.40825700759887695, + "grad_norm": 23.12116241455078, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.3093240261077881, + "logits/rejected": -0.316268652677536, + "logps/chosen": -260.2626953125, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -466.9383544921875, + "loss": 1.0894, + "margin_dpo/margin_mean": 161.9666748046875, + "margin_dpo/margin_std": 203.50302124023438, + "step": 275 + }, + { + "KL/chosen_KL_mean": -241.7791290283203, + "KL/mean": -322.0965576171875, + "KL/rejected_KL_mean": -402.4139404296875, + "KL/std": 190.34759521484375, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.0024003759026527405, + "fcm_dpo/delta": 0.02319333702325821, + "fcm_dpo/margin": 160.63482666015625, + "fcm_dpo/q_t": 0.4141028821468353, + "grad_norm": 29.356189727783203, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.23649513721466064, + "logits/rejected": -0.20997360348701477, + "logps/chosen": -298.3565368652344, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -472.77960205078125, + "loss": 1.1107, + "margin_dpo/margin_mean": 160.63482666015625, + "margin_dpo/margin_std": 233.87583923339844, + "step": 276 + }, + { + "KL/chosen_KL_mean": -263.56640625, + "KL/mean": -349.49713134765625, + "KL/rejected_KL_mean": -435.42779541015625, + "KL/std": 192.81695556640625, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.002432500245049596, + "fcm_dpo/delta": -0.032088592648506165, + "fcm_dpo/margin": 171.86138916015625, + "fcm_dpo/q_t": 0.4016081690788269, + "grad_norm": 26.00898551940918, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.26711416244506836, + "logits/rejected": -0.24979907274246216, + "logps/chosen": -319.8379821777344, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -528.30908203125, + "loss": 1.0858, + "margin_dpo/margin_mean": 171.86138916015625, + "margin_dpo/margin_std": 228.77777099609375, + "step": 277 + }, + { + "KL/chosen_KL_mean": -235.44161987304688, + "KL/mean": -336.4327392578125, + "KL/rejected_KL_mean": -437.42388916015625, + "KL/std": 201.9891357421875, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.002094062976539135, + "fcm_dpo/delta": -0.045741938054561615, + "fcm_dpo/margin": 201.98226928710938, + "fcm_dpo/q_t": 0.3976041078567505, + "grad_norm": 22.05190658569336, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.24980834126472473, + "logits/rejected": -0.25416260957717896, + "logps/chosen": -288.383544921875, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -528.677490234375, + "loss": 1.0605, + "margin_dpo/margin_mean": 201.9822998046875, + "margin_dpo/margin_std": 226.97244262695312, + "step": 278 + }, + { + "KL/chosen_KL_mean": -271.568115234375, + "KL/mean": -371.5149841308594, + "KL/rejected_KL_mean": -471.46185302734375, + "KL/std": 216.6588897705078, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.0019927441608160734, + "fcm_dpo/delta": -0.005218621343374252, + "fcm_dpo/margin": 199.89370727539062, + "fcm_dpo/q_t": 0.40749794244766235, + "grad_norm": 26.85765838623047, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.19662827253341675, + "logits/rejected": -0.19574469327926636, + "logps/chosen": -320.20947265625, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -559.3132934570312, + "loss": 1.0963, + "margin_dpo/margin_mean": 199.89370727539062, + "margin_dpo/margin_std": 265.7958984375, + "step": 279 + }, + { + "KL/chosen_KL_mean": -264.79876708984375, + "KL/mean": -365.7987060546875, + "KL/rejected_KL_mean": -466.79864501953125, + "KL/std": 195.9139862060547, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.0020362103823572397, + "fcm_dpo/delta": -0.01925138756632805, + "fcm_dpo/margin": 201.9998779296875, + "fcm_dpo/q_t": 0.4014127254486084, + "grad_norm": 20.498315811157227, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.24768587946891785, + "logits/rejected": -0.24966652691364288, + "logps/chosen": -323.59588623046875, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -565.41748046875, + "loss": 1.0628, + "margin_dpo/margin_mean": 201.99989318847656, + "margin_dpo/margin_std": 216.71530151367188, + "step": 280 + }, + { + "KL/chosen_KL_mean": -247.58489990234375, + "KL/mean": -335.40484619140625, + "KL/rejected_KL_mean": -423.2247619628906, + "KL/std": 196.88302612304688, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0021747334394603968, + "fcm_dpo/delta": 0.022978195920586586, + "fcm_dpo/margin": 175.6398162841797, + "fcm_dpo/q_t": 0.4103030562400818, + "grad_norm": 21.528810501098633, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.3240095376968384, + "logits/rejected": -0.3004445433616638, + "logps/chosen": -303.07342529296875, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -504.1073303222656, + "loss": 1.0948, + "margin_dpo/margin_mean": 175.6398162841797, + "margin_dpo/margin_std": 213.6923828125, + "step": 281 + }, + { + "KL/chosen_KL_mean": -268.12432861328125, + "KL/mean": -337.4071044921875, + "KL/rejected_KL_mean": -406.68988037109375, + "KL/std": 196.039794921875, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.0021595642901957035, + "fcm_dpo/delta": 0.02335960976779461, + "fcm_dpo/margin": 138.56552124023438, + "fcm_dpo/q_t": 0.4331875145435333, + "grad_norm": 21.117961883544922, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.3865717053413391, + "logits/rejected": -0.3700195550918579, + "logps/chosen": -341.1944885253906, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -502.0408935546875, + "loss": 1.1627, + "margin_dpo/margin_mean": 138.56552124023438, + "margin_dpo/margin_std": 225.9492645263672, + "step": 282 + }, + { + "KL/chosen_KL_mean": -283.1618347167969, + "KL/mean": -388.3852233886719, + "KL/rejected_KL_mean": -493.608642578125, + "KL/std": 241.67697143554688, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.0020841769874095917, + "fcm_dpo/delta": -0.06936129927635193, + "fcm_dpo/margin": 210.44679260253906, + "fcm_dpo/q_t": 0.3913659453392029, + "grad_norm": 29.183977127075195, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.33291661739349365, + "logits/rejected": -0.3372425436973572, + "logps/chosen": -345.0602722167969, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -590.59521484375, + "loss": 1.057, + "margin_dpo/margin_mean": 210.44679260253906, + "margin_dpo/margin_std": 231.2135009765625, + "step": 283 + }, + { + "KL/chosen_KL_mean": -270.32830810546875, + "KL/mean": -369.215087890625, + "KL/rejected_KL_mean": -468.1018371582031, + "KL/std": 219.54708862304688, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0019610195886343718, + "fcm_dpo/delta": 0.01984664611518383, + "fcm_dpo/margin": 197.77349853515625, + "fcm_dpo/q_t": 0.4114644527435303, + "grad_norm": 25.379192352294922, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.37090370059013367, + "logits/rejected": -0.3533616065979004, + "logps/chosen": -328.76385498046875, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -561.5711059570312, + "loss": 1.0843, + "margin_dpo/margin_mean": 197.77349853515625, + "margin_dpo/margin_std": 232.99339294433594, + "step": 284 + }, + { + "KL/chosen_KL_mean": -313.6859130859375, + "KL/mean": -400.13983154296875, + "KL/rejected_KL_mean": -486.59375, + "KL/std": 192.8905029296875, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.002196455840021372, + "fcm_dpo/delta": 0.021015789359807968, + "fcm_dpo/margin": 172.90782165527344, + "fcm_dpo/q_t": 0.41028088331222534, + "grad_norm": 23.027700424194336, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.36354196071624756, + "logits/rejected": -0.3486790955066681, + "logps/chosen": -379.9181213378906, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -585.7205810546875, + "loss": 1.1035, + "margin_dpo/margin_mean": 172.9078369140625, + "margin_dpo/margin_std": 218.47084045410156, + "step": 285 + }, + { + "KL/chosen_KL_mean": -327.0200500488281, + "KL/mean": -422.5919189453125, + "KL/rejected_KL_mean": -518.163818359375, + "KL/std": 230.7489776611328, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.002110408851876855, + "fcm_dpo/delta": -0.006586743518710136, + "fcm_dpo/margin": 191.14376831054688, + "fcm_dpo/q_t": 0.4062705934047699, + "grad_norm": 23.25826644897461, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.35514867305755615, + "logits/rejected": -0.3393225073814392, + "logps/chosen": -399.9710693359375, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -606.7522583007812, + "loss": 1.0915, + "margin_dpo/margin_mean": 191.14376831054688, + "margin_dpo/margin_std": 254.77488708496094, + "step": 286 + }, + { + "KL/chosen_KL_mean": -300.4835510253906, + "KL/mean": -379.34942626953125, + "KL/rejected_KL_mean": -458.21527099609375, + "KL/std": 201.38296508789062, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.0022800066508352757, + "fcm_dpo/delta": 0.06143353134393692, + "fcm_dpo/margin": 157.731689453125, + "fcm_dpo/q_t": 0.42126035690307617, + "grad_norm": 27.160978317260742, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.30109989643096924, + "logits/rejected": -0.27528104186058044, + "logps/chosen": -362.02471923828125, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -535.9113159179688, + "loss": 1.1154, + "margin_dpo/margin_mean": 157.731689453125, + "margin_dpo/margin_std": 216.85557556152344, + "step": 287 + }, + { + "KL/chosen_KL_mean": -276.4866943359375, + "KL/mean": -374.0993347167969, + "KL/rejected_KL_mean": -471.71197509765625, + "KL/std": 193.3370361328125, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.002058174926787615, + "fcm_dpo/delta": -0.026749789714813232, + "fcm_dpo/margin": 195.2252655029297, + "fcm_dpo/q_t": 0.40044307708740234, + "grad_norm": 21.408998489379883, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.34782230854034424, + "logits/rejected": -0.3332597613334656, + "logps/chosen": -333.1479187011719, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -559.0476684570312, + "loss": 1.0755, + "margin_dpo/margin_mean": 195.2252655029297, + "margin_dpo/margin_std": 204.141357421875, + "step": 288 + }, + { + "KL/chosen_KL_mean": -259.8858337402344, + "KL/mean": -367.64276123046875, + "KL/rejected_KL_mean": -475.3996887207031, + "KL/std": 213.38723754882812, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.0020438190549612045, + "fcm_dpo/delta": -0.07187651097774506, + "fcm_dpo/margin": 215.51380920410156, + "fcm_dpo/q_t": 0.3916912078857422, + "grad_norm": 24.203519821166992, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.3048250675201416, + "logits/rejected": -0.3042747378349304, + "logps/chosen": -305.1162414550781, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -563.0423583984375, + "loss": 1.0505, + "margin_dpo/margin_mean": 215.51380920410156, + "margin_dpo/margin_std": 233.81417846679688, + "step": 289 + }, + { + "KL/chosen_KL_mean": -269.5323791503906, + "KL/mean": -379.81329345703125, + "KL/rejected_KL_mean": -490.09423828125, + "KL/std": 221.88650512695312, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.0018460073042660952, + "fcm_dpo/delta": -0.013119934126734734, + "fcm_dpo/margin": 220.56182861328125, + "fcm_dpo/q_t": 0.4045451879501343, + "grad_norm": 20.667232513427734, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.2957327961921692, + "logits/rejected": -0.31704530119895935, + "logps/chosen": -325.0038757324219, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -606.8027954101562, + "loss": 1.0762, + "margin_dpo/margin_mean": 220.56182861328125, + "margin_dpo/margin_std": 266.3393859863281, + "step": 290 + }, + { + "KL/chosen_KL_mean": -240.43844604492188, + "KL/mean": -341.17254638671875, + "KL/rejected_KL_mean": -441.9066467285156, + "KL/std": 196.33567810058594, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.001962371403351426, + "fcm_dpo/delta": 0.0012248605489730835, + "fcm_dpo/margin": 201.46820068359375, + "fcm_dpo/q_t": 0.40641385316848755, + "grad_norm": 25.23223114013672, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3080219626426697, + "logits/rejected": -0.29609572887420654, + "logps/chosen": -303.69879150390625, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -531.2037353515625, + "loss": 1.0851, + "margin_dpo/margin_mean": 201.46820068359375, + "margin_dpo/margin_std": 237.0089874267578, + "step": 291 + }, + { + "KL/chosen_KL_mean": -267.9251708984375, + "KL/mean": -372.0924987792969, + "KL/rejected_KL_mean": -476.25982666015625, + "KL/std": 229.24436950683594, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.0019257422536611557, + "fcm_dpo/delta": -0.005529198795557022, + "fcm_dpo/margin": 208.33460998535156, + "fcm_dpo/q_t": 0.40545937418937683, + "grad_norm": 21.263906478881836, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.3315708637237549, + "logits/rejected": -0.3213496208190918, + "logps/chosen": -321.8437194824219, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -566.22119140625, + "loss": 1.0907, + "margin_dpo/margin_mean": 208.33460998535156, + "margin_dpo/margin_std": 271.5143737792969, + "step": 292 + }, + { + "KL/chosen_KL_mean": -278.5457763671875, + "KL/mean": -362.8836975097656, + "KL/rejected_KL_mean": -447.22161865234375, + "KL/std": 229.81143188476562, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.002093075541779399, + "fcm_dpo/delta": 0.06820766627788544, + "fcm_dpo/margin": 168.67581176757812, + "fcm_dpo/q_t": 0.42393064498901367, + "grad_norm": 32.55642318725586, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.3256397843360901, + "logits/rejected": -0.3210792541503906, + "logps/chosen": -338.92181396484375, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -525.0740356445312, + "loss": 1.1334, + "margin_dpo/margin_mean": 168.67581176757812, + "margin_dpo/margin_std": 262.6167907714844, + "step": 293 + }, + { + "KL/chosen_KL_mean": -259.8635559082031, + "KL/mean": -352.86517333984375, + "KL/rejected_KL_mean": -445.8667907714844, + "KL/std": 204.66827392578125, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.002107615116983652, + "fcm_dpo/delta": 0.012143617495894432, + "fcm_dpo/margin": 186.0032196044922, + "fcm_dpo/q_t": 0.4118291139602661, + "grad_norm": 22.22089195251465, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.3044741451740265, + "logits/rejected": -0.311285138130188, + "logps/chosen": -307.95111083984375, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -527.7637939453125, + "loss": 1.1023, + "margin_dpo/margin_mean": 186.0032196044922, + "margin_dpo/margin_std": 260.2086181640625, + "step": 294 + }, + { + "KL/chosen_KL_mean": -320.80810546875, + "KL/mean": -423.1761474609375, + "KL/rejected_KL_mean": -525.5442504882812, + "KL/std": 238.36337280273438, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.002073909156024456, + "fcm_dpo/delta": -0.04264184087514877, + "fcm_dpo/margin": 204.73614501953125, + "fcm_dpo/q_t": 0.4011147618293762, + "grad_norm": 27.336824417114258, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.3063458800315857, + "logits/rejected": -0.31039565801620483, + "logps/chosen": -370.7327880859375, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -613.0005493164062, + "loss": 1.0927, + "margin_dpo/margin_mean": 204.73614501953125, + "margin_dpo/margin_std": 289.61798095703125, + "step": 295 + }, + { + "KL/chosen_KL_mean": -394.3350830078125, + "KL/mean": -464.76324462890625, + "KL/rejected_KL_mean": -535.19140625, + "KL/std": 229.95870971679688, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.0020087361335754395, + "fcm_dpo/delta": 0.00424933061003685, + "fcm_dpo/margin": 140.85630798339844, + "fcm_dpo/q_t": 0.4354059398174286, + "grad_norm": 33.60810852050781, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.3529552221298218, + "logits/rejected": -0.3394961953163147, + "logps/chosen": -459.8262939453125, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -628.2804565429688, + "loss": 1.2109, + "margin_dpo/margin_mean": 140.85629272460938, + "margin_dpo/margin_std": 299.87567138671875, + "step": 296 + }, + { + "KL/chosen_KL_mean": -344.30731201171875, + "KL/mean": -423.664794921875, + "KL/rejected_KL_mean": -503.0222473144531, + "KL/std": 215.01690673828125, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.002223505638539791, + "fcm_dpo/delta": 0.07282247394323349, + "fcm_dpo/margin": 158.7149200439453, + "fcm_dpo/q_t": 0.4236125946044922, + "grad_norm": 42.23075485229492, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.3576158583164215, + "logits/rejected": -0.36280083656311035, + "logps/chosen": -400.7842712402344, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -598.1607666015625, + "loss": 1.1244, + "margin_dpo/margin_mean": 158.71493530273438, + "margin_dpo/margin_std": 226.28025817871094, + "step": 297 + }, + { + "KL/chosen_KL_mean": -358.29290771484375, + "KL/mean": -466.8505859375, + "KL/rejected_KL_mean": -575.408203125, + "KL/std": 281.4233093261719, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.0020931584294885397, + "fcm_dpo/delta": -0.0989355742931366, + "fcm_dpo/margin": 217.11532592773438, + "fcm_dpo/q_t": 0.39393287897109985, + "grad_norm": 29.66607093811035, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.3804464340209961, + "logits/rejected": -0.39784708619117737, + "logps/chosen": -425.61810302734375, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -692.0704345703125, + "loss": 1.0936, + "margin_dpo/margin_mean": 217.1153106689453, + "margin_dpo/margin_std": 334.7890930175781, + "step": 298 + }, + { + "KL/chosen_KL_mean": -290.83984375, + "KL/mean": -384.0028076171875, + "KL/rejected_KL_mean": -477.165771484375, + "KL/std": 211.8370361328125, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.0020763694774359465, + "fcm_dpo/delta": 0.006563708186149597, + "fcm_dpo/margin": 186.32595825195312, + "fcm_dpo/q_t": 0.41148841381073, + "grad_norm": 29.92995262145996, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.35604220628738403, + "logits/rejected": -0.3601064085960388, + "logps/chosen": -339.80194091796875, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -561.4940185546875, + "loss": 1.1168, + "margin_dpo/margin_mean": 186.32595825195312, + "margin_dpo/margin_std": 269.77178955078125, + "step": 299 + }, + { + "KL/chosen_KL_mean": -361.88922119140625, + "KL/mean": -479.45281982421875, + "KL/rejected_KL_mean": -597.016357421875, + "KL/std": 256.1278381347656, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0018352132756263018, + "fcm_dpo/delta": -0.055538419634103775, + "fcm_dpo/margin": 235.1271514892578, + "fcm_dpo/q_t": 0.3963469862937927, + "grad_norm": 22.620500564575195, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.35153138637542725, + "logits/rejected": -0.3557186424732208, + "logps/chosen": -420.96295166015625, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -692.9827880859375, + "loss": 1.0815, + "margin_dpo/margin_mean": 235.12716674804688, + "margin_dpo/margin_std": 311.29791259765625, + "step": 300 + }, + { + "KL/chosen_KL_mean": -296.9462890625, + "KL/mean": -410.1385498046875, + "KL/rejected_KL_mean": -523.3307495117188, + "KL/std": 233.12229919433594, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.0017267554067075253, + "fcm_dpo/delta": 0.014057676307857037, + "fcm_dpo/margin": 226.38442993164062, + "fcm_dpo/q_t": 0.4108361601829529, + "grad_norm": 18.437192916870117, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.3524608612060547, + "logits/rejected": -0.3491095006465912, + "logps/chosen": -354.1956787109375, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -615.684326171875, + "loss": 1.0883, + "margin_dpo/margin_mean": 226.38442993164062, + "margin_dpo/margin_std": 280.14007568359375, + "step": 301 + }, + { + "KL/chosen_KL_mean": -269.1903076171875, + "KL/mean": -361.92718505859375, + "KL/rejected_KL_mean": -454.6640625, + "KL/std": 212.72869873046875, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.001909639802761376, + "fcm_dpo/delta": 0.07072426378726959, + "fcm_dpo/margin": 185.4737548828125, + "fcm_dpo/q_t": 0.42330536246299744, + "grad_norm": 20.071674346923828, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.34278440475463867, + "logits/rejected": -0.34728020429611206, + "logps/chosen": -320.3883056640625, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -551.8904418945312, + "loss": 1.1251, + "margin_dpo/margin_mean": 185.4737548828125, + "margin_dpo/margin_std": 256.4941101074219, + "step": 302 + }, + { + "KL/chosen_KL_mean": -279.39471435546875, + "KL/mean": -365.64605712890625, + "KL/rejected_KL_mean": -451.89739990234375, + "KL/std": 218.69338989257812, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.0021402090787887573, + "fcm_dpo/delta": 0.04843229800462723, + "fcm_dpo/margin": 172.50267028808594, + "fcm_dpo/q_t": 0.41948962211608887, + "grad_norm": 23.985990524291992, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.38367849588394165, + "logits/rejected": -0.3712496757507324, + "logps/chosen": -346.1086730957031, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -538.8428344726562, + "loss": 1.1289, + "margin_dpo/margin_mean": 172.50267028808594, + "margin_dpo/margin_std": 265.6961669921875, + "step": 303 + }, + { + "KL/chosen_KL_mean": -255.20101928710938, + "KL/mean": -347.8723449707031, + "KL/rejected_KL_mean": -440.54364013671875, + "KL/std": 201.79244995117188, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.0022568381391465664, + "fcm_dpo/delta": -0.03559745475649834, + "fcm_dpo/margin": 185.34263610839844, + "fcm_dpo/q_t": 0.3972512185573578, + "grad_norm": 40.94980239868164, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.40435200929641724, + "logits/rejected": -0.39599835872650146, + "logps/chosen": -327.1517333984375, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -531.0156860351562, + "loss": 1.0579, + "margin_dpo/margin_mean": 185.34262084960938, + "margin_dpo/margin_std": 180.45718383789062, + "step": 304 + }, + { + "KL/chosen_KL_mean": -266.6454772949219, + "KL/mean": -351.7230224609375, + "KL/rejected_KL_mean": -436.8005676269531, + "KL/std": 221.42990112304688, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.0021582874469459057, + "fcm_dpo/delta": 0.04888404905796051, + "fcm_dpo/margin": 170.1551055908203, + "fcm_dpo/q_t": 0.41974300146102905, + "grad_norm": 22.135257720947266, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.4137483537197113, + "logits/rejected": -0.39837193489074707, + "logps/chosen": -333.4407043457031, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -529.55517578125, + "loss": 1.1212, + "margin_dpo/margin_mean": 170.1551055908203, + "margin_dpo/margin_std": 243.96319580078125, + "step": 305 + }, + { + "KL/chosen_KL_mean": -261.8374938964844, + "KL/mean": -343.0050354003906, + "KL/rejected_KL_mean": -424.1725769042969, + "KL/std": 194.5423583984375, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.002403826918452978, + "fcm_dpo/delta": 0.015124435536563396, + "fcm_dpo/margin": 162.33505249023438, + "fcm_dpo/q_t": 0.4113726019859314, + "grad_norm": 23.578460693359375, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.3788735866546631, + "logits/rejected": -0.35274261236190796, + "logps/chosen": -331.5213928222656, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -509.3317565917969, + "loss": 1.0925, + "margin_dpo/margin_mean": 162.33505249023438, + "margin_dpo/margin_std": 210.62339782714844, + "step": 306 + }, + { + "KL/chosen_KL_mean": -237.50689697265625, + "KL/mean": -322.65093994140625, + "KL/rejected_KL_mean": -407.7949523925781, + "KL/std": 183.3559112548828, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.0023515899665653706, + "fcm_dpo/delta": -0.0012738704681396484, + "fcm_dpo/margin": 170.28805541992188, + "fcm_dpo/q_t": 0.40794315934181213, + "grad_norm": 24.69138526916504, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.37078455090522766, + "logits/rejected": -0.35139191150665283, + "logps/chosen": -307.67230224609375, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -494.76727294921875, + "loss": 1.0939, + "margin_dpo/margin_mean": 170.28805541992188, + "margin_dpo/margin_std": 231.32247924804688, + "step": 307 + }, + { + "KL/chosen_KL_mean": -242.2062225341797, + "KL/mean": -339.5050964355469, + "KL/rejected_KL_mean": -436.803955078125, + "KL/std": 205.6837158203125, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.002198460977524519, + "fcm_dpo/delta": -0.049677349627017975, + "fcm_dpo/margin": 194.59776306152344, + "fcm_dpo/q_t": 0.3950149416923523, + "grad_norm": 29.672807693481445, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.3967479467391968, + "logits/rejected": -0.3871362507343292, + "logps/chosen": -297.45123291015625, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -516.1762084960938, + "loss": 1.05, + "margin_dpo/margin_mean": 194.59774780273438, + "margin_dpo/margin_std": 198.84432983398438, + "step": 308 + }, + { + "KL/chosen_KL_mean": -249.229248046875, + "KL/mean": -345.9730224609375, + "KL/rejected_KL_mean": -442.71673583984375, + "KL/std": 216.7500457763672, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.00216893432661891, + "fcm_dpo/delta": -0.040406569838523865, + "fcm_dpo/margin": 193.48745727539062, + "fcm_dpo/q_t": 0.3975731432437897, + "grad_norm": 26.32208251953125, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.32539457082748413, + "logits/rejected": -0.3188524842262268, + "logps/chosen": -298.1883544921875, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -525.0574951171875, + "loss": 1.0796, + "margin_dpo/margin_mean": 193.4874725341797, + "margin_dpo/margin_std": 231.7923583984375, + "step": 309 + }, + { + "KL/chosen_KL_mean": -287.5223388671875, + "KL/mean": -375.1282958984375, + "KL/rejected_KL_mean": -462.7342224121094, + "KL/std": 191.27525329589844, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.002167648635804653, + "fcm_dpo/delta": 0.02439631149172783, + "fcm_dpo/margin": 175.21188354492188, + "fcm_dpo/q_t": 0.41132253408432007, + "grad_norm": 37.333168029785156, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.37113621830940247, + "logits/rejected": -0.35927319526672363, + "logps/chosen": -350.2641296386719, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -542.66455078125, + "loss": 1.0851, + "margin_dpo/margin_mean": 175.21188354492188, + "margin_dpo/margin_std": 188.38232421875, + "step": 310 + }, + { + "KL/chosen_KL_mean": -305.24285888671875, + "KL/mean": -408.00628662109375, + "KL/rejected_KL_mean": -510.7697448730469, + "KL/std": 250.8982696533203, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.002006505150347948, + "fcm_dpo/delta": -0.021621335297822952, + "fcm_dpo/margin": 205.52688598632812, + "fcm_dpo/q_t": 0.40415090322494507, + "grad_norm": 23.370407104492188, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.3467109501361847, + "logits/rejected": -0.32372719049453735, + "logps/chosen": -358.2708435058594, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -588.2079467773438, + "loss": 1.0815, + "margin_dpo/margin_mean": 205.52688598632812, + "margin_dpo/margin_std": 265.2549743652344, + "step": 311 + }, + { + "KL/chosen_KL_mean": -289.9778747558594, + "KL/mean": -387.76654052734375, + "KL/rejected_KL_mean": -485.55523681640625, + "KL/std": 229.4159393310547, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0020645607728511095, + "fcm_dpo/delta": -0.011410795152187347, + "fcm_dpo/margin": 195.57733154296875, + "fcm_dpo/q_t": 0.4080260396003723, + "grad_norm": 22.71463394165039, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.2678723633289337, + "logits/rejected": -0.2562296390533447, + "logps/chosen": -339.3700866699219, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -561.3480224609375, + "loss": 1.0977, + "margin_dpo/margin_mean": 195.57733154296875, + "margin_dpo/margin_std": 275.55340576171875, + "step": 312 + }, + { + "KL/chosen_KL_mean": -276.6165771484375, + "KL/mean": -375.31329345703125, + "KL/rejected_KL_mean": -474.01007080078125, + "KL/std": 237.2071533203125, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.0020586608443409204, + "fcm_dpo/delta": -0.02704712748527527, + "fcm_dpo/margin": 197.3934326171875, + "fcm_dpo/q_t": 0.40283501148223877, + "grad_norm": 26.37751579284668, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.3565303385257721, + "logits/rejected": -0.3560790419578552, + "logps/chosen": -326.76934814453125, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -560.416259765625, + "loss": 1.1012, + "margin_dpo/margin_mean": 197.3934326171875, + "margin_dpo/margin_std": 258.5960693359375, + "step": 313 + }, + { + "KL/chosen_KL_mean": -297.798828125, + "KL/mean": -391.349853515625, + "KL/rejected_KL_mean": -484.90087890625, + "KL/std": 228.58383178710938, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0019475116860121489, + "fcm_dpo/delta": 0.055439822375774384, + "fcm_dpo/margin": 187.10211181640625, + "fcm_dpo/q_t": 0.4213699698448181, + "grad_norm": 22.879201889038086, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.39260512590408325, + "logits/rejected": -0.39466455578804016, + "logps/chosen": -355.036376953125, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -582.4974365234375, + "loss": 1.1347, + "margin_dpo/margin_mean": 187.10211181640625, + "margin_dpo/margin_std": 293.64080810546875, + "step": 314 + }, + { + "KL/chosen_KL_mean": -271.39031982421875, + "KL/mean": -353.8436279296875, + "KL/rejected_KL_mean": -436.296875, + "KL/std": 202.65625, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.0022210520692169666, + "fcm_dpo/delta": 0.05339854210615158, + "fcm_dpo/margin": 164.90655517578125, + "fcm_dpo/q_t": 0.41980287432670593, + "grad_norm": 24.21739959716797, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.3449453115463257, + "logits/rejected": -0.3495122194290161, + "logps/chosen": -320.45989990234375, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -521.977783203125, + "loss": 1.1145, + "margin_dpo/margin_mean": 164.90655517578125, + "margin_dpo/margin_std": 229.23095703125, + "step": 315 + }, + { + "KL/chosen_KL_mean": -270.35205078125, + "KL/mean": -393.6520690917969, + "KL/rejected_KL_mean": -516.9520263671875, + "KL/std": 250.11061096191406, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0017885593697428703, + "fcm_dpo/delta": -0.11213979125022888, + "fcm_dpo/margin": 246.60000610351562, + "fcm_dpo/q_t": 0.3872528076171875, + "grad_norm": 22.613483428955078, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.40570446848869324, + "logits/rejected": -0.40869593620300293, + "logps/chosen": -324.6128234863281, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -618.2335205078125, + "loss": 1.0531, + "margin_dpo/margin_mean": 246.60000610351562, + "margin_dpo/margin_std": 279.0342102050781, + "step": 316 + }, + { + "KL/chosen_KL_mean": -269.0587158203125, + "KL/mean": -400.5162658691406, + "KL/rejected_KL_mean": -531.9737548828125, + "KL/std": 222.888427734375, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.0017005549743771553, + "fcm_dpo/delta": -0.08523563295602798, + "fcm_dpo/margin": 262.91510009765625, + "fcm_dpo/q_t": 0.3873101472854614, + "grad_norm": 26.969892501831055, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.3938244581222534, + "logits/rejected": -0.3835000693798065, + "logps/chosen": -325.1529235839844, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -632.6728515625, + "loss": 1.0363, + "margin_dpo/margin_mean": 262.91510009765625, + "margin_dpo/margin_std": 250.01504516601562, + "step": 317 + }, + { + "KL/chosen_KL_mean": -297.5737609863281, + "KL/mean": -394.74908447265625, + "KL/rejected_KL_mean": -491.9244384765625, + "KL/std": 229.82669067382812, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0017898095538839698, + "fcm_dpo/delta": 0.07254049926996231, + "fcm_dpo/margin": 194.35072326660156, + "fcm_dpo/q_t": 0.42293781042099, + "grad_norm": 21.67091178894043, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.45766058564186096, + "logits/rejected": -0.4315134584903717, + "logps/chosen": -362.2194519042969, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -574.688720703125, + "loss": 1.1147, + "margin_dpo/margin_mean": 194.35073852539062, + "margin_dpo/margin_std": 247.84156799316406, + "step": 318 + }, + { + "KL/chosen_KL_mean": -268.0478820800781, + "KL/mean": -386.1382751464844, + "KL/rejected_KL_mean": -504.2286682128906, + "KL/std": 255.11056518554688, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.0017438679933547974, + "fcm_dpo/delta": -0.02019650489091873, + "fcm_dpo/margin": 236.18075561523438, + "fcm_dpo/q_t": 0.40276533365249634, + "grad_norm": 24.052961349487305, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.40444713830947876, + "logits/rejected": -0.4185525178909302, + "logps/chosen": -317.431640625, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -618.1351318359375, + "loss": 1.0723, + "margin_dpo/margin_mean": 236.18075561523438, + "margin_dpo/margin_std": 282.4808349609375, + "step": 319 + }, + { + "KL/chosen_KL_mean": -281.9687194824219, + "KL/mean": -408.4984130859375, + "KL/rejected_KL_mean": -535.0281372070312, + "KL/std": 261.2070007324219, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.0016149728326126933, + "fcm_dpo/delta": -0.019406653940677643, + "fcm_dpo/margin": 253.0594482421875, + "fcm_dpo/q_t": 0.40273725986480713, + "grad_norm": 27.48386001586914, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.39280545711517334, + "logits/rejected": -0.397488534450531, + "logps/chosen": -341.4736022949219, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -632.6953125, + "loss": 1.0693, + "margin_dpo/margin_mean": 253.05943298339844, + "margin_dpo/margin_std": 285.04510498046875, + "step": 320 + }, + { + "KL/chosen_KL_mean": -347.18927001953125, + "KL/mean": -454.4679260253906, + "KL/rejected_KL_mean": -561.74658203125, + "KL/std": 268.3890686035156, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.0017120807897299528, + "fcm_dpo/delta": 0.0495997853577137, + "fcm_dpo/margin": 214.55735778808594, + "fcm_dpo/q_t": 0.41874903440475464, + "grad_norm": 21.842844009399414, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.3386973738670349, + "logits/rejected": -0.3240874409675598, + "logps/chosen": -408.7379150390625, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -653.3876953125, + "loss": 1.1147, + "margin_dpo/margin_mean": 214.55735778808594, + "margin_dpo/margin_std": 290.58978271484375, + "step": 321 + }, + { + "KL/chosen_KL_mean": -306.6402893066406, + "KL/mean": -424.40594482421875, + "KL/rejected_KL_mean": -542.171630859375, + "KL/std": 241.84188842773438, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.0017245409544557333, + "fcm_dpo/delta": -0.015119031071662903, + "fcm_dpo/margin": 235.53131103515625, + "fcm_dpo/q_t": 0.4033818244934082, + "grad_norm": 24.67373275756836, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.3635835349559784, + "logits/rejected": -0.36071500182151794, + "logps/chosen": -363.9303283691406, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -637.9215698242188, + "loss": 1.0722, + "margin_dpo/margin_mean": 235.5313262939453, + "margin_dpo/margin_std": 266.08941650390625, + "step": 322 + }, + { + "KL/chosen_KL_mean": -327.5718994140625, + "KL/mean": -426.56634521484375, + "KL/rejected_KL_mean": -525.560791015625, + "KL/std": 240.03286743164062, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.0018759286031126976, + "fcm_dpo/delta": 0.04566890001296997, + "fcm_dpo/margin": 197.98890686035156, + "fcm_dpo/q_t": 0.41689807176589966, + "grad_norm": 30.36820411682129, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.391653448343277, + "logits/rejected": -0.3899046778678894, + "logps/chosen": -378.8058776855469, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -600.6227416992188, + "loss": 1.1077, + "margin_dpo/margin_mean": 197.98890686035156, + "margin_dpo/margin_std": 260.3305969238281, + "step": 323 + }, + { + "KL/chosen_KL_mean": -376.3759765625, + "KL/mean": -470.93865966796875, + "KL/rejected_KL_mean": -565.5013427734375, + "KL/std": 256.7306823730469, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.002103945007547736, + "fcm_dpo/delta": -0.02135651558637619, + "fcm_dpo/margin": 189.12530517578125, + "fcm_dpo/q_t": 0.41041427850723267, + "grad_norm": 50.850955963134766, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.3911237120628357, + "logits/rejected": -0.37464720010757446, + "logps/chosen": -441.51116943359375, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -651.9788818359375, + "loss": 1.1359, + "margin_dpo/margin_mean": 189.12530517578125, + "margin_dpo/margin_std": 302.71319580078125, + "step": 324 + }, + { + "KL/chosen_KL_mean": -296.9546203613281, + "KL/mean": -379.6038513183594, + "KL/rejected_KL_mean": -462.25311279296875, + "KL/std": 226.74127197265625, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.002113199792802334, + "fcm_dpo/delta": 0.0753774344921112, + "fcm_dpo/margin": 165.2984619140625, + "fcm_dpo/q_t": 0.4240267872810364, + "grad_norm": 27.387161254882812, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.4187164306640625, + "logits/rejected": -0.3953896760940552, + "logps/chosen": -353.17022705078125, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -532.3390502929688, + "loss": 1.1291, + "margin_dpo/margin_mean": 165.2984619140625, + "margin_dpo/margin_std": 237.98086547851562, + "step": 325 + }, + { + "KL/chosen_KL_mean": -324.411865234375, + "KL/mean": -408.02496337890625, + "KL/rejected_KL_mean": -491.6380920410156, + "KL/std": 210.09243774414062, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0022600889205932617, + "fcm_dpo/delta": 0.03556614741683006, + "fcm_dpo/margin": 167.2262420654297, + "fcm_dpo/q_t": 0.414311021566391, + "grad_norm": 37.292301177978516, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.44619232416152954, + "logits/rejected": -0.42077717185020447, + "logps/chosen": -397.1368408203125, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -571.48486328125, + "loss": 1.0978, + "margin_dpo/margin_mean": 167.2262420654297, + "margin_dpo/margin_std": 205.65249633789062, + "step": 326 + }, + { + "KL/chosen_KL_mean": -280.4002685546875, + "KL/mean": -404.8081970214844, + "KL/rejected_KL_mean": -529.2161254882812, + "KL/std": 223.6777801513672, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0018240961944684386, + "fcm_dpo/delta": -0.12385020405054092, + "fcm_dpo/margin": 248.81582641601562, + "fcm_dpo/q_t": 0.3799220621585846, + "grad_norm": 32.990352630615234, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.41425788402557373, + "logits/rejected": -0.4029478132724762, + "logps/chosen": -349.53472900390625, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -641.14990234375, + "loss": 1.0343, + "margin_dpo/margin_mean": 248.81582641601562, + "margin_dpo/margin_std": 254.188720703125, + "step": 327 + }, + { + "KL/chosen_KL_mean": -289.2528381347656, + "KL/mean": -397.5467224121094, + "KL/rejected_KL_mean": -505.84063720703125, + "KL/std": 236.40066528320312, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0018365057185292244, + "fcm_dpo/delta": 0.003600106341764331, + "fcm_dpo/margin": 216.58779907226562, + "fcm_dpo/q_t": 0.4082072973251343, + "grad_norm": 25.665807723999023, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.35888296365737915, + "logits/rejected": -0.36398810148239136, + "logps/chosen": -348.9400329589844, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -596.6956176757812, + "loss": 1.0842, + "margin_dpo/margin_mean": 216.58779907226562, + "margin_dpo/margin_std": 268.95428466796875, + "step": 328 + }, + { + "KL/chosen_KL_mean": -328.11090087890625, + "KL/mean": -442.0875244140625, + "KL/rejected_KL_mean": -556.0640869140625, + "KL/std": 269.01434326171875, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.0017261970788240433, + "fcm_dpo/delta": 0.0008079372346401215, + "fcm_dpo/margin": 227.95321655273438, + "fcm_dpo/q_t": 0.40766558051109314, + "grad_norm": 24.41579818725586, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.3722694516181946, + "logits/rejected": -0.3635803461074829, + "logps/chosen": -393.35711669921875, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -656.7618408203125, + "loss": 1.1045, + "margin_dpo/margin_mean": 227.95321655273438, + "margin_dpo/margin_std": 301.1634521484375, + "step": 329 + }, + { + "KL/chosen_KL_mean": -275.129150390625, + "KL/mean": -394.5272216796875, + "KL/rejected_KL_mean": -513.92529296875, + "KL/std": 246.2072296142578, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0017482815310359001, + "fcm_dpo/delta": -0.03028486669063568, + "fcm_dpo/margin": 238.79615783691406, + "fcm_dpo/q_t": 0.400304913520813, + "grad_norm": 20.50057029724121, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.3955565094947815, + "logits/rejected": -0.39654165506362915, + "logps/chosen": -322.12750244140625, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -600.8021240234375, + "loss": 1.0642, + "margin_dpo/margin_mean": 238.79615783691406, + "margin_dpo/margin_std": 271.3484191894531, + "step": 330 + }, + { + "KL/chosen_KL_mean": -305.6733093261719, + "KL/mean": -425.43841552734375, + "KL/rejected_KL_mean": -545.2034912109375, + "KL/std": 206.7650146484375, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.0017030672170221806, + "fcm_dpo/delta": -0.013409378007054329, + "fcm_dpo/margin": 239.53012084960938, + "fcm_dpo/q_t": 0.40146273374557495, + "grad_norm": 22.680070877075195, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.37283700704574585, + "logits/rejected": -0.3688333034515381, + "logps/chosen": -356.19757080078125, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -634.2188720703125, + "loss": 1.0483, + "margin_dpo/margin_mean": 239.5301513671875, + "margin_dpo/margin_std": 205.19898986816406, + "step": 331 + }, + { + "KL/chosen_KL_mean": -309.19793701171875, + "KL/mean": -390.8100891113281, + "KL/rejected_KL_mean": -472.4222412109375, + "KL/std": 211.47320556640625, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.0020041917450726032, + "fcm_dpo/delta": 0.10211435705423355, + "fcm_dpo/margin": 163.22427368164062, + "fcm_dpo/q_t": 0.43064454197883606, + "grad_norm": 23.14905548095703, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.360334187746048, + "logits/rejected": -0.33985477685928345, + "logps/chosen": -358.37823486328125, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -548.9073486328125, + "loss": 1.139, + "margin_dpo/margin_mean": 163.22427368164062, + "margin_dpo/margin_std": 238.90945434570312, + "step": 332 + }, + { + "KL/chosen_KL_mean": -336.2814636230469, + "KL/mean": -449.5226135253906, + "KL/rejected_KL_mean": -562.7637939453125, + "KL/std": 258.14825439453125, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.001787829678505659, + "fcm_dpo/delta": -0.0357210636138916, + "fcm_dpo/margin": 226.48231506347656, + "fcm_dpo/q_t": 0.4021400809288025, + "grad_norm": 37.613563537597656, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.34511464834213257, + "logits/rejected": -0.3372688889503479, + "logps/chosen": -400.0372314453125, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -657.807861328125, + "loss": 1.1071, + "margin_dpo/margin_mean": 226.48233032226562, + "margin_dpo/margin_std": 314.77203369140625, + "step": 333 + }, + { + "KL/chosen_KL_mean": -324.66705322265625, + "KL/mean": -422.1019287109375, + "KL/rejected_KL_mean": -519.5368041992188, + "KL/std": 276.22686767578125, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0019669593311846256, + "fcm_dpo/delta": 0.0270000621676445, + "fcm_dpo/margin": 194.86973571777344, + "fcm_dpo/q_t": 0.41571369767189026, + "grad_norm": 21.602262496948242, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.38413316011428833, + "logits/rejected": -0.3805462718009949, + "logps/chosen": -391.6468200683594, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -614.853759765625, + "loss": 1.1271, + "margin_dpo/margin_mean": 194.86972045898438, + "margin_dpo/margin_std": 309.7537536621094, + "step": 334 + }, + { + "KL/chosen_KL_mean": -354.06585693359375, + "KL/mean": -434.64276123046875, + "KL/rejected_KL_mean": -515.2196655273438, + "KL/std": 258.8491516113281, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.002282127272337675, + "fcm_dpo/delta": 0.03639799728989601, + "fcm_dpo/margin": 161.15380859375, + "fcm_dpo/q_t": 0.41994214057922363, + "grad_norm": 31.488386154174805, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.41236335039138794, + "logits/rejected": -0.39202946424484253, + "logps/chosen": -416.60833740234375, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -602.83740234375, + "loss": 1.1504, + "margin_dpo/margin_mean": 161.15379333496094, + "margin_dpo/margin_std": 279.69989013671875, + "step": 335 + }, + { + "KL/chosen_KL_mean": -347.204345703125, + "KL/mean": -475.4479064941406, + "KL/rejected_KL_mean": -603.69140625, + "KL/std": 295.57818603515625, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.0018394170328974724, + "fcm_dpo/delta": -0.1347188502550125, + "fcm_dpo/margin": 256.4870910644531, + "fcm_dpo/q_t": 0.3836958408355713, + "grad_norm": 27.45176887512207, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.38979193568229675, + "logits/rejected": -0.3902202248573303, + "logps/chosen": -401.7355041503906, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -708.095703125, + "loss": 1.0657, + "margin_dpo/margin_mean": 256.4870910644531, + "margin_dpo/margin_std": 350.75457763671875, + "step": 336 + }, + { + "KL/chosen_KL_mean": -316.01263427734375, + "KL/mean": -441.3327941894531, + "KL/rejected_KL_mean": -566.6529541015625, + "KL/std": 243.086181640625, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.001608079532161355, + "fcm_dpo/delta": -0.006862752139568329, + "fcm_dpo/margin": 250.64028930664062, + "fcm_dpo/q_t": 0.4053046405315399, + "grad_norm": 20.08615493774414, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.41743162274360657, + "logits/rejected": -0.40591827034950256, + "logps/chosen": -381.1413269042969, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -668.3799438476562, + "loss": 1.0743, + "margin_dpo/margin_mean": 250.6403045654297, + "margin_dpo/margin_std": 288.3606872558594, + "step": 337 + }, + { + "KL/chosen_KL_mean": -287.96209716796875, + "KL/mean": -410.0386962890625, + "KL/rejected_KL_mean": -532.1152954101562, + "KL/std": 230.76153564453125, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.0016326969489455223, + "fcm_dpo/delta": 0.002140873111784458, + "fcm_dpo/margin": 244.1531524658203, + "fcm_dpo/q_t": 0.4057929515838623, + "grad_norm": 22.323871612548828, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.42750677466392517, + "logits/rejected": -0.4222105145454407, + "logps/chosen": -346.3847961425781, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -621.183837890625, + "loss": 1.0645, + "margin_dpo/margin_mean": 244.15316772460938, + "margin_dpo/margin_std": 244.08584594726562, + "step": 338 + }, + { + "KL/chosen_KL_mean": -303.424560546875, + "KL/mean": -419.2534484863281, + "KL/rejected_KL_mean": -535.0823974609375, + "KL/std": 239.62149047851562, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0017491846811026335, + "fcm_dpo/delta": -0.029546871781349182, + "fcm_dpo/margin": 231.65774536132812, + "fcm_dpo/q_t": 0.40035557746887207, + "grad_norm": 24.648235321044922, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.43223291635513306, + "logits/rejected": -0.42046141624450684, + "logps/chosen": -363.4198913574219, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -638.9932861328125, + "loss": 1.0808, + "margin_dpo/margin_mean": 231.65774536132812, + "margin_dpo/margin_std": 242.7178497314453, + "step": 339 + }, + { + "KL/chosen_KL_mean": -324.7750244140625, + "KL/mean": -426.1435546875, + "KL/rejected_KL_mean": -527.5120239257812, + "KL/std": 234.62620544433594, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0017105863662436604, + "fcm_dpo/delta": 0.08201248943805695, + "fcm_dpo/margin": 202.73699951171875, + "fcm_dpo/q_t": 0.425212025642395, + "grad_norm": 34.08943176269531, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.3556247353553772, + "logits/rejected": -0.33346790075302124, + "logps/chosen": -377.60528564453125, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -600.6192626953125, + "loss": 1.1282, + "margin_dpo/margin_mean": 202.73699951171875, + "margin_dpo/margin_std": 283.2412109375, + "step": 340 + }, + { + "KL/chosen_KL_mean": -331.2342224121094, + "KL/mean": -450.0869140625, + "KL/rejected_KL_mean": -568.9395751953125, + "KL/std": 259.69732666015625, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0017094245413318276, + "fcm_dpo/delta": -0.016936201602220535, + "fcm_dpo/margin": 237.70541381835938, + "fcm_dpo/q_t": 0.4038848578929901, + "grad_norm": 25.37321662902832, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.37913063168525696, + "logits/rejected": -0.37905052304267883, + "logps/chosen": -379.134033203125, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -670.74951171875, + "loss": 1.0753, + "margin_dpo/margin_mean": 237.70538330078125, + "margin_dpo/margin_std": 275.2982177734375, + "step": 341 + }, + { + "KL/chosen_KL_mean": -320.9766540527344, + "KL/mean": -426.2503967285156, + "KL/rejected_KL_mean": -531.524169921875, + "KL/std": 245.80764770507812, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0018736084457486868, + "fcm_dpo/delta": 0.0011573433876037598, + "fcm_dpo/margin": 210.54747009277344, + "fcm_dpo/q_t": 0.4087076187133789, + "grad_norm": 27.59446144104004, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.45016488432884216, + "logits/rejected": -0.42709416151046753, + "logps/chosen": -392.9732971191406, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -624.11376953125, + "loss": 1.1059, + "margin_dpo/margin_mean": 210.5474853515625, + "margin_dpo/margin_std": 294.087158203125, + "step": 342 + }, + { + "KL/chosen_KL_mean": -307.4436950683594, + "KL/mean": -433.9495849609375, + "KL/rejected_KL_mean": -560.4554443359375, + "KL/std": 248.22842407226562, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.001662734430283308, + "fcm_dpo/delta": -0.036752138286828995, + "fcm_dpo/margin": 253.01181030273438, + "fcm_dpo/q_t": 0.3963671922683716, + "grad_norm": 21.48690414428711, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.4163956344127655, + "logits/rejected": -0.4220476746559143, + "logps/chosen": -361.84930419921875, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -671.4969482421875, + "loss": 1.042, + "margin_dpo/margin_mean": 253.01181030273438, + "margin_dpo/margin_std": 219.65103149414062, + "step": 343 + }, + { + "KL/chosen_KL_mean": -313.14569091796875, + "KL/mean": -424.7760925292969, + "KL/rejected_KL_mean": -536.406494140625, + "KL/std": 274.6375427246094, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.0015421407297253609, + "fcm_dpo/delta": -0.06822478026151657, + "fcm_dpo/margin": 223.26084899902344, + "fcm_dpo/q_t": 0.41599759459495544, + "grad_norm": 17.046581268310547, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.3974248170852661, + "logits/rejected": -0.39894139766693115, + "logps/chosen": -367.1103515625, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -627.0298461914062, + "loss": 1.1263, + "margin_dpo/margin_mean": 223.26084899902344, + "margin_dpo/margin_std": 279.2683410644531, + "step": 344 + }, + { + "KL/chosen_KL_mean": -370.6162414550781, + "KL/mean": -496.4261474609375, + "KL/rejected_KL_mean": -622.236083984375, + "KL/std": 260.423095703125, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.001530057517811656, + "fcm_dpo/delta": 0.020667294040322304, + "fcm_dpo/margin": 251.619873046875, + "fcm_dpo/q_t": 0.4105876684188843, + "grad_norm": 20.95164680480957, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.42145538330078125, + "logits/rejected": -0.41829943656921387, + "logps/chosen": -432.30194091796875, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -721.7265014648438, + "loss": 1.084, + "margin_dpo/margin_mean": 251.619873046875, + "margin_dpo/margin_std": 285.166259765625, + "step": 345 + }, + { + "KL/chosen_KL_mean": -365.87664794921875, + "KL/mean": -480.89178466796875, + "KL/rejected_KL_mean": -595.906982421875, + "KL/std": 254.52523803710938, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.001609054277651012, + "fcm_dpo/delta": 0.04702121764421463, + "fcm_dpo/margin": 230.03033447265625, + "fcm_dpo/q_t": 0.415949285030365, + "grad_norm": 20.622108459472656, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.3826184570789337, + "logits/rejected": -0.37241339683532715, + "logps/chosen": -421.1329040527344, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -673.322265625, + "loss": 1.0974, + "margin_dpo/margin_mean": 230.03033447265625, + "margin_dpo/margin_std": 276.1398620605469, + "step": 346 + }, + { + "KL/chosen_KL_mean": -361.59759521484375, + "KL/mean": -479.4732666015625, + "KL/rejected_KL_mean": -597.348876953125, + "KL/std": 272.4322509765625, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.0016747593181207776, + "fcm_dpo/delta": 0.00839798804372549, + "fcm_dpo/margin": 235.7513427734375, + "fcm_dpo/q_t": 0.4093061089515686, + "grad_norm": 21.934925079345703, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.4126300811767578, + "logits/rejected": -0.40708625316619873, + "logps/chosen": -419.163818359375, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -689.7039794921875, + "loss": 1.0905, + "margin_dpo/margin_mean": 235.7513427734375, + "margin_dpo/margin_std": 303.5050964355469, + "step": 347 + }, + { + "KL/chosen_KL_mean": -326.0738220214844, + "KL/mean": -424.66363525390625, + "KL/rejected_KL_mean": -523.25341796875, + "KL/std": 231.39590454101562, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.001778720528818667, + "fcm_dpo/delta": 0.07379812002182007, + "fcm_dpo/margin": 197.17959594726562, + "fcm_dpo/q_t": 0.42386046051979065, + "grad_norm": 24.249563217163086, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.44509440660476685, + "logits/rejected": -0.44437336921691895, + "logps/chosen": -382.39154052734375, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -612.391845703125, + "loss": 1.1286, + "margin_dpo/margin_mean": 197.1796112060547, + "margin_dpo/margin_std": 269.8843994140625, + "step": 348 + }, + { + "KL/chosen_KL_mean": -308.29888916015625, + "KL/mean": -417.86456298828125, + "KL/rejected_KL_mean": -527.4302978515625, + "KL/std": 253.47750854492188, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.0019209685269743204, + "fcm_dpo/delta": -0.03923729062080383, + "fcm_dpo/margin": 219.1314239501953, + "fcm_dpo/q_t": 0.40259507298469543, + "grad_norm": 23.687149047851562, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.45386502146720886, + "logits/rejected": -0.45663371682167053, + "logps/chosen": -366.32440185546875, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -624.9354248046875, + "loss": 1.0951, + "margin_dpo/margin_mean": 219.13143920898438, + "margin_dpo/margin_std": 312.9615478515625, + "step": 349 + }, + { + "KL/chosen_KL_mean": -332.3165283203125, + "KL/mean": -443.63629150390625, + "KL/rejected_KL_mean": -554.9560546875, + "KL/std": 248.83816528320312, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.0017454567132517695, + "fcm_dpo/delta": 0.013433754444122314, + "fcm_dpo/margin": 222.63954162597656, + "fcm_dpo/q_t": 0.4100583791732788, + "grad_norm": 26.530868530273438, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.4750177562236786, + "logits/rejected": -0.4820059835910797, + "logps/chosen": -396.64703369140625, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -644.8277587890625, + "loss": 1.1064, + "margin_dpo/margin_mean": 222.63955688476562, + "margin_dpo/margin_std": 307.46893310546875, + "step": 350 + }, + { + "KL/chosen_KL_mean": -296.4862060546875, + "KL/mean": -429.1023254394531, + "KL/rejected_KL_mean": -561.718505859375, + "KL/std": 283.80633544921875, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0016240214463323355, + "fcm_dpo/delta": -0.061422545462846756, + "fcm_dpo/margin": 265.2322998046875, + "fcm_dpo/q_t": 0.3965432643890381, + "grad_norm": 31.153547286987305, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.4499565362930298, + "logits/rejected": -0.4501733183860779, + "logps/chosen": -357.1583251953125, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -663.283935546875, + "loss": 1.0671, + "margin_dpo/margin_mean": 265.2322998046875, + "margin_dpo/margin_std": 327.75909423828125, + "step": 351 + }, + { + "KL/chosen_KL_mean": -349.35406494140625, + "KL/mean": -436.3336181640625, + "KL/rejected_KL_mean": -523.313232421875, + "KL/std": 263.14996337890625, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.0018782506231218576, + "fcm_dpo/delta": 0.10981118679046631, + "fcm_dpo/margin": 173.9591064453125, + "fcm_dpo/q_t": 0.4318041503429413, + "grad_norm": 46.439903259277344, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.45800480246543884, + "logits/rejected": -0.4245471954345703, + "logps/chosen": -420.2975158691406, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -599.9552001953125, + "loss": 1.155, + "margin_dpo/margin_mean": 173.9591064453125, + "margin_dpo/margin_std": 280.7679748535156, + "step": 352 + }, + { + "KL/chosen_KL_mean": -317.61041259765625, + "KL/mean": -430.0752258300781, + "KL/rejected_KL_mean": -542.5400390625, + "KL/std": 259.1496276855469, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.0018013006774708629, + "fcm_dpo/delta": -0.02872556447982788, + "fcm_dpo/margin": 224.9296112060547, + "fcm_dpo/q_t": 0.40363016724586487, + "grad_norm": 23.421403884887695, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.4782019555568695, + "logits/rejected": -0.4654806852340698, + "logps/chosen": -376.0057373046875, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -622.8755493164062, + "loss": 1.0947, + "margin_dpo/margin_mean": 224.92959594726562, + "margin_dpo/margin_std": 295.62744140625, + "step": 353 + }, + { + "KL/chosen_KL_mean": -272.10107421875, + "KL/mean": -383.7615966796875, + "KL/rejected_KL_mean": -495.422119140625, + "KL/std": 226.24905395507812, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.001837678486481309, + "fcm_dpo/delta": -0.01862015575170517, + "fcm_dpo/margin": 223.32101440429688, + "fcm_dpo/q_t": 0.4018193483352661, + "grad_norm": 24.753582000732422, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.45796817541122437, + "logits/rejected": -0.4484265446662903, + "logps/chosen": -331.9040832519531, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -584.1796264648438, + "loss": 1.0623, + "margin_dpo/margin_mean": 223.32101440429688, + "margin_dpo/margin_std": 235.79531860351562, + "step": 354 + }, + { + "KL/chosen_KL_mean": -267.42645263671875, + "KL/mean": -380.90118408203125, + "KL/rejected_KL_mean": -494.37591552734375, + "KL/std": 236.37852478027344, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.0017689072992652655, + "fcm_dpo/delta": -0.005881348624825478, + "fcm_dpo/margin": 226.94949340820312, + "fcm_dpo/q_t": 0.40436455607414246, + "grad_norm": 38.343528747558594, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.48708575963974, + "logits/rejected": -0.4752500653266907, + "logps/chosen": -321.554931640625, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -576.781982421875, + "loss": 1.0671, + "margin_dpo/margin_mean": 226.9495086669922, + "margin_dpo/margin_std": 235.01458740234375, + "step": 355 + }, + { + "KL/chosen_KL_mean": -329.96331787109375, + "KL/mean": -393.64111328125, + "KL/rejected_KL_mean": -457.3189697265625, + "KL/std": 250.74232482910156, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.00188022933434695, + "fcm_dpo/delta": 0.029842089861631393, + "fcm_dpo/margin": 127.35562896728516, + "fcm_dpo/q_t": 0.4473668932914734, + "grad_norm": 28.56047248840332, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.47035759687423706, + "logits/rejected": -0.44759708642959595, + "logps/chosen": -394.6371154785156, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -533.21826171875, + "loss": 1.2407, + "margin_dpo/margin_mean": 127.35563659667969, + "margin_dpo/margin_std": 307.847412109375, + "step": 356 + }, + { + "KL/chosen_KL_mean": -290.78863525390625, + "KL/mean": -393.8023681640625, + "KL/rejected_KL_mean": -496.8161315917969, + "KL/std": 240.8219757080078, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.0018875239184126258, + "fcm_dpo/delta": 0.013352084904909134, + "fcm_dpo/margin": 206.02752685546875, + "fcm_dpo/q_t": 0.4096463918685913, + "grad_norm": 27.79951286315918, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.45739173889160156, + "logits/rejected": -0.44537484645843506, + "logps/chosen": -343.514404296875, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -583.6572875976562, + "loss": 1.0937, + "margin_dpo/margin_mean": 206.02752685546875, + "margin_dpo/margin_std": 253.822509765625, + "step": 357 + }, + { + "KL/chosen_KL_mean": -269.3954162597656, + "KL/mean": -368.242919921875, + "KL/rejected_KL_mean": -467.09039306640625, + "KL/std": 235.274658203125, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0020065298303961754, + "fcm_dpo/delta": 0.005489196628332138, + "fcm_dpo/margin": 197.69497680664062, + "fcm_dpo/q_t": 0.410032719373703, + "grad_norm": 28.569393157958984, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.48102307319641113, + "logits/rejected": -0.463295042514801, + "logps/chosen": -332.6008605957031, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -555.4637451171875, + "loss": 1.0932, + "margin_dpo/margin_mean": 197.69496154785156, + "margin_dpo/margin_std": 264.3070983886719, + "step": 358 + }, + { + "KL/chosen_KL_mean": -304.4889221191406, + "KL/mean": -419.318359375, + "KL/rejected_KL_mean": -534.1477661132812, + "KL/std": 241.59100341796875, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0018390321638435125, + "fcm_dpo/delta": -0.04261445254087448, + "fcm_dpo/margin": 229.6588134765625, + "fcm_dpo/q_t": 0.3999358117580414, + "grad_norm": 27.864505767822266, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.4233545958995819, + "logits/rejected": -0.40909135341644287, + "logps/chosen": -360.859130859375, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -616.321533203125, + "loss": 1.0733, + "margin_dpo/margin_mean": 229.6588134765625, + "margin_dpo/margin_std": 284.9479064941406, + "step": 359 + }, + { + "KL/chosen_KL_mean": -301.6378479003906, + "KL/mean": -391.5552978515625, + "KL/rejected_KL_mean": -481.4727478027344, + "KL/std": 206.68411254882812, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.0020297402516007423, + "fcm_dpo/delta": 0.05278204008936882, + "fcm_dpo/margin": 179.8349151611328, + "fcm_dpo/q_t": 0.4177628755569458, + "grad_norm": 40.045162200927734, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.43576139211654663, + "logits/rejected": -0.422180712223053, + "logps/chosen": -353.0982360839844, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -551.3116455078125, + "loss": 1.0958, + "margin_dpo/margin_mean": 179.83489990234375, + "margin_dpo/margin_std": 205.51583862304688, + "step": 360 + }, + { + "KL/chosen_KL_mean": -332.83001708984375, + "KL/mean": -420.3536376953125, + "KL/rejected_KL_mean": -507.8772277832031, + "KL/std": 242.87753295898438, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0021930173970758915, + "fcm_dpo/delta": 0.021988222375512123, + "fcm_dpo/margin": 175.04721069335938, + "fcm_dpo/q_t": 0.4129222631454468, + "grad_norm": 37.02199172973633, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.4316215217113495, + "logits/rejected": -0.427333801984787, + "logps/chosen": -386.69952392578125, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -598.646484375, + "loss": 1.115, + "margin_dpo/margin_mean": 175.04721069335938, + "margin_dpo/margin_std": 257.5013427734375, + "step": 361 + }, + { + "KL/chosen_KL_mean": -281.8548583984375, + "KL/mean": -425.60888671875, + "KL/rejected_KL_mean": -569.3629150390625, + "KL/std": 254.64712524414062, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.0015978740993887186, + "fcm_dpo/delta": -0.14796875417232513, + "fcm_dpo/margin": 287.50799560546875, + "fcm_dpo/q_t": 0.37716490030288696, + "grad_norm": 23.246320724487305, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.4437373876571655, + "logits/rejected": -0.45072150230407715, + "logps/chosen": -340.49395751953125, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -674.94482421875, + "loss": 1.0236, + "margin_dpo/margin_mean": 287.50799560546875, + "margin_dpo/margin_std": 274.0611572265625, + "step": 362 + }, + { + "KL/chosen_KL_mean": -271.1270751953125, + "KL/mean": -413.6645202636719, + "KL/rejected_KL_mean": -556.2019653320312, + "KL/std": 249.9840087890625, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0015145809156820178, + "fcm_dpo/delta": -0.056021787226200104, + "fcm_dpo/margin": 285.07489013671875, + "fcm_dpo/q_t": 0.39367860555648804, + "grad_norm": 20.8101749420166, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.43102937936782837, + "logits/rejected": -0.4243617057800293, + "logps/chosen": -315.6854553222656, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -630.8969116210938, + "loss": 1.0437, + "margin_dpo/margin_mean": 285.07489013671875, + "margin_dpo/margin_std": 276.6128234863281, + "step": 363 + }, + { + "KL/chosen_KL_mean": -300.5224914550781, + "KL/mean": -406.80938720703125, + "KL/rejected_KL_mean": -513.0963134765625, + "KL/std": 243.1720733642578, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0016487597022205591, + "fcm_dpo/delta": 0.06513446569442749, + "fcm_dpo/margin": 212.57382202148438, + "fcm_dpo/q_t": 0.42112141847610474, + "grad_norm": 21.724754333496094, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.44869840145111084, + "logits/rejected": -0.4586051106452942, + "logps/chosen": -349.4171142578125, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -604.4920654296875, + "loss": 1.1204, + "margin_dpo/margin_mean": 212.57383728027344, + "margin_dpo/margin_std": 284.5193176269531, + "step": 364 + }, + { + "KL/chosen_KL_mean": -304.2714538574219, + "KL/mean": -416.9530029296875, + "KL/rejected_KL_mean": -529.634521484375, + "KL/std": 258.83819580078125, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0016951878787949681, + "fcm_dpo/delta": 0.02898254618048668, + "fcm_dpo/margin": 225.36306762695312, + "fcm_dpo/q_t": 0.41397157311439514, + "grad_norm": 20.870635986328125, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.43810129165649414, + "logits/rejected": -0.44645947217941284, + "logps/chosen": -355.7641906738281, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -622.336181640625, + "loss": 1.0909, + "margin_dpo/margin_mean": 225.36306762695312, + "margin_dpo/margin_std": 272.38427734375, + "step": 365 + }, + { + "KL/chosen_KL_mean": -287.7502746582031, + "KL/mean": -404.9134826660156, + "KL/rejected_KL_mean": -522.0767211914062, + "KL/std": 256.9874267578125, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0016629542224109173, + "fcm_dpo/delta": 0.010643713176250458, + "fcm_dpo/margin": 234.326416015625, + "fcm_dpo/q_t": 0.4102753698825836, + "grad_norm": 19.262928009033203, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.4263162612915039, + "logits/rejected": -0.44171929359436035, + "logps/chosen": -332.4708557128906, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -605.3870849609375, + "loss": 1.0992, + "margin_dpo/margin_mean": 234.326416015625, + "margin_dpo/margin_std": 304.7642517089844, + "step": 366 + }, + { + "KL/chosen_KL_mean": -303.0289611816406, + "KL/mean": -399.514892578125, + "KL/rejected_KL_mean": -496.0008544921875, + "KL/std": 225.67916870117188, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0019213332561776042, + "fcm_dpo/delta": 0.04300880804657936, + "fcm_dpo/margin": 192.971923828125, + "fcm_dpo/q_t": 0.4157758355140686, + "grad_norm": 21.952829360961914, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.48216262459754944, + "logits/rejected": -0.4676123857498169, + "logps/chosen": -361.43438720703125, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -572.752197265625, + "loss": 1.1045, + "margin_dpo/margin_mean": 192.971923828125, + "margin_dpo/margin_std": 250.1029052734375, + "step": 367 + }, + { + "KL/chosen_KL_mean": -282.49639892578125, + "KL/mean": -436.16912841796875, + "KL/rejected_KL_mean": -589.8418579101562, + "KL/std": 253.77154541015625, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0015261652879416943, + "fcm_dpo/delta": -0.14009416103363037, + "fcm_dpo/margin": 307.345458984375, + "fcm_dpo/q_t": 0.37267598509788513, + "grad_norm": 34.17594528198242, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.4614467918872833, + "logits/rejected": -0.4720000922679901, + "logps/chosen": -326.94891357421875, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -688.3970947265625, + "loss": 0.9994, + "margin_dpo/margin_mean": 307.345458984375, + "margin_dpo/margin_std": 224.4804229736328, + "step": 368 + }, + { + "KL/chosen_KL_mean": -367.19921875, + "KL/mean": -449.5543212890625, + "KL/rejected_KL_mean": -531.909423828125, + "KL/std": 254.9052276611328, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0015358589589595795, + "fcm_dpo/delta": 0.05729483440518379, + "fcm_dpo/margin": 164.710205078125, + "fcm_dpo/q_t": 0.4428786039352417, + "grad_norm": 24.15337371826172, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.4904573857784271, + "logits/rejected": -0.4741830825805664, + "logps/chosen": -438.58074951171875, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -623.2052001953125, + "loss": 1.1956, + "margin_dpo/margin_mean": 164.710205078125, + "margin_dpo/margin_std": 289.94000244140625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -390.330810546875, + "KL/mean": -462.7243347167969, + "KL/rejected_KL_mean": -535.1178588867188, + "KL/std": 265.82781982421875, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0017272934783250093, + "fcm_dpo/delta": 0.06001188978552818, + "fcm_dpo/margin": 144.7870635986328, + "fcm_dpo/q_t": 0.44495099782943726, + "grad_norm": 28.623899459838867, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.4887716770172119, + "logits/rejected": -0.4825999140739441, + "logps/chosen": -461.93829345703125, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -632.377685546875, + "loss": 1.2167, + "margin_dpo/margin_mean": 144.7870635986328, + "margin_dpo/margin_std": 307.05999755859375, + "step": 370 + }, + { + "KL/chosen_KL_mean": -373.00421142578125, + "KL/mean": -484.24078369140625, + "KL/rejected_KL_mean": -595.477294921875, + "KL/std": 274.64886474609375, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0018450857605785131, + "fcm_dpo/delta": -0.020257379859685898, + "fcm_dpo/margin": 222.4730987548828, + "fcm_dpo/q_t": 0.4056010842323303, + "grad_norm": 29.032577514648438, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.5206031799316406, + "logits/rejected": -0.5084042549133301, + "logps/chosen": -442.418701171875, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -694.6495361328125, + "loss": 1.1008, + "margin_dpo/margin_mean": 222.4730987548828, + "margin_dpo/margin_std": 321.52008056640625, + "step": 371 + }, + { + "KL/chosen_KL_mean": -360.9215393066406, + "KL/mean": -495.5597229003906, + "KL/rejected_KL_mean": -630.1979370117188, + "KL/std": 315.78912353515625, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.001595328445546329, + "fcm_dpo/delta": -0.05453532934188843, + "fcm_dpo/margin": 269.2763671875, + "fcm_dpo/q_t": 0.39929690957069397, + "grad_norm": 26.51140594482422, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.4976493716239929, + "logits/rejected": -0.4843401610851288, + "logps/chosen": -422.7395324707031, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -708.7374267578125, + "loss": 1.0767, + "margin_dpo/margin_mean": 269.2763671875, + "margin_dpo/margin_std": 363.7861328125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -385.86328125, + "KL/mean": -511.8228759765625, + "KL/rejected_KL_mean": -637.7823486328125, + "KL/std": 299.7609558105469, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.001618098933249712, + "fcm_dpo/delta": -0.01854773610830307, + "fcm_dpo/margin": 251.91912841796875, + "fcm_dpo/q_t": 0.40265533328056335, + "grad_norm": 23.972576141357422, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.46941909193992615, + "logits/rejected": -0.4492412209510803, + "logps/chosen": -450.0804138183594, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -723.7420043945312, + "loss": 1.0837, + "margin_dpo/margin_mean": 251.91912841796875, + "margin_dpo/margin_std": 299.7061767578125, + "step": 373 + }, + { + "KL/chosen_KL_mean": -349.892333984375, + "KL/mean": -465.20263671875, + "KL/rejected_KL_mean": -580.512939453125, + "KL/std": 325.4590148925781, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.0015723302494734526, + "fcm_dpo/delta": 0.05567358061671257, + "fcm_dpo/margin": 230.62063598632812, + "fcm_dpo/q_t": 0.42263877391815186, + "grad_norm": 22.6434383392334, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.5414470434188843, + "logits/rejected": -0.524927020072937, + "logps/chosen": -420.54254150390625, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -674.1531372070312, + "loss": 1.1406, + "margin_dpo/margin_mean": 230.62063598632812, + "margin_dpo/margin_std": 373.28961181640625, + "step": 374 + }, + { + "KL/chosen_KL_mean": -364.77288818359375, + "KL/mean": -473.304931640625, + "KL/rejected_KL_mean": -581.8369750976562, + "KL/std": 262.2808837890625, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0017019468359649181, + "fcm_dpo/delta": 0.04178474843502045, + "fcm_dpo/margin": 217.06411743164062, + "fcm_dpo/q_t": 0.41703087091445923, + "grad_norm": 31.42391586303711, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.4924180507659912, + "logits/rejected": -0.4902787208557129, + "logps/chosen": -424.85308837890625, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -670.7752685546875, + "loss": 1.1287, + "margin_dpo/margin_mean": 217.06411743164062, + "margin_dpo/margin_std": 315.9866943359375, + "step": 375 + }, + { + "KL/chosen_KL_mean": -347.1155090332031, + "KL/mean": -484.00567626953125, + "KL/rejected_KL_mean": -620.8958740234375, + "KL/std": 285.42181396484375, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0016071631107479334, + "fcm_dpo/delta": -0.07716603577136993, + "fcm_dpo/margin": 273.7803955078125, + "fcm_dpo/q_t": 0.39170658588409424, + "grad_norm": 24.55660629272461, + "learning_rate": 2.5e-07, + "logits/chosen": -0.49682873487472534, + "logits/rejected": -0.48542171716690063, + "logps/chosen": -409.77581787109375, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -726.4224853515625, + "loss": 1.0607, + "margin_dpo/margin_mean": 273.7803955078125, + "margin_dpo/margin_std": 332.501708984375, + "step": 376 + }, + { + "KL/chosen_KL_mean": -349.8211975097656, + "KL/mean": -483.8100280761719, + "KL/rejected_KL_mean": -617.7988891601562, + "KL/std": 303.7757263183594, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0015564009081572294, + "fcm_dpo/delta": -0.030124623328447342, + "fcm_dpo/margin": 267.9776916503906, + "fcm_dpo/q_t": 0.401019811630249, + "grad_norm": 21.409746170043945, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.4336928129196167, + "logits/rejected": -0.432683527469635, + "logps/chosen": -404.2999267578125, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -716.502197265625, + "loss": 1.072, + "margin_dpo/margin_mean": 267.9776611328125, + "margin_dpo/margin_std": 321.41864013671875, + "step": 377 + }, + { + "KL/chosen_KL_mean": -324.5780029296875, + "KL/mean": -463.517333984375, + "KL/rejected_KL_mean": -602.4566650390625, + "KL/std": 269.98370361328125, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.0014617801643908024, + "fcm_dpo/delta": -0.010532225482165813, + "fcm_dpo/margin": 277.878662109375, + "fcm_dpo/q_t": 0.40280961990356445, + "grad_norm": 24.20557403564453, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.43811696767807007, + "logits/rejected": -0.4507744312286377, + "logps/chosen": -369.5985107421875, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -690.503662109375, + "loss": 1.0649, + "margin_dpo/margin_mean": 277.878662109375, + "margin_dpo/margin_std": 294.4283447265625, + "step": 378 + }, + { + "KL/chosen_KL_mean": -359.47418212890625, + "KL/mean": -495.9452209472656, + "KL/rejected_KL_mean": -632.416259765625, + "KL/std": 277.9902038574219, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0014259706949815154, + "fcm_dpo/delta": 0.0134861059486866, + "fcm_dpo/margin": 272.94207763671875, + "fcm_dpo/q_t": 0.41009724140167236, + "grad_norm": 26.755817413330078, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.38554373383522034, + "logits/rejected": -0.4059056043624878, + "logps/chosen": -412.65625, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -746.7164306640625, + "loss": 1.0908, + "margin_dpo/margin_mean": 272.9421081542969, + "margin_dpo/margin_std": 334.01593017578125, + "step": 379 + }, + { + "KL/chosen_KL_mean": -353.51654052734375, + "KL/mean": -504.0564270019531, + "KL/rejected_KL_mean": -654.5963134765625, + "KL/std": 314.81463623046875, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0014171417569741607, + "fcm_dpo/delta": -0.046321943402290344, + "fcm_dpo/margin": 301.0798034667969, + "fcm_dpo/q_t": 0.39565783739089966, + "grad_norm": 22.568410873413086, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.4838710427284241, + "logits/rejected": -0.5105802416801453, + "logps/chosen": -404.86956787109375, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -758.7880249023438, + "loss": 1.0598, + "margin_dpo/margin_mean": 301.079833984375, + "margin_dpo/margin_std": 341.14447021484375, + "step": 380 + }, + { + "KL/chosen_KL_mean": -365.193359375, + "KL/mean": -466.3520812988281, + "KL/rejected_KL_mean": -567.5108642578125, + "KL/std": 262.38165283203125, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.001597442664206028, + "fcm_dpo/delta": 0.11401475220918655, + "fcm_dpo/margin": 202.31747436523438, + "fcm_dpo/q_t": 0.43302297592163086, + "grad_norm": 25.53712272644043, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4542039632797241, + "logits/rejected": -0.4321713149547577, + "logps/chosen": -422.99639892578125, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -646.730224609375, + "loss": 1.1718, + "margin_dpo/margin_mean": 202.31747436523438, + "margin_dpo/margin_std": 359.52716064453125, + "step": 381 + }, + { + "KL/chosen_KL_mean": -346.081298828125, + "KL/mean": -476.01800537109375, + "KL/rejected_KL_mean": -605.9547119140625, + "KL/std": 252.94512939453125, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0016131680458784103, + "fcm_dpo/delta": -0.03418818861246109, + "fcm_dpo/margin": 259.8733825683594, + "fcm_dpo/q_t": 0.3991495966911316, + "grad_norm": 20.419843673706055, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.5054559707641602, + "logits/rejected": -0.5107045769691467, + "logps/chosen": -412.10162353515625, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -716.6648559570312, + "loss": 1.0605, + "margin_dpo/margin_mean": 259.8733825683594, + "margin_dpo/margin_std": 285.6113586425781, + "step": 382 + }, + { + "KL/chosen_KL_mean": -361.154541015625, + "KL/mean": -490.5584716796875, + "KL/rejected_KL_mean": -619.96240234375, + "KL/std": 283.15301513671875, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0015578651800751686, + "fcm_dpo/delta": -0.00645710714161396, + "fcm_dpo/margin": 258.8077392578125, + "fcm_dpo/q_t": 0.4064059555530548, + "grad_norm": 27.733203887939453, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.4914437532424927, + "logits/rejected": -0.5008552670478821, + "logps/chosen": -411.5460510253906, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -713.67822265625, + "loss": 1.1016, + "margin_dpo/margin_mean": 258.8077392578125, + "margin_dpo/margin_std": 370.54718017578125, + "step": 383 + }, + { + "KL/chosen_KL_mean": -375.8470458984375, + "KL/mean": -478.16192626953125, + "KL/rejected_KL_mean": -580.4768676757812, + "KL/std": 256.4949645996094, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0017840572400018573, + "fcm_dpo/delta": 0.04306403547525406, + "fcm_dpo/margin": 204.6298065185547, + "fcm_dpo/q_t": 0.41565489768981934, + "grad_norm": 26.314607620239258, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.501458466053009, + "logits/rejected": -0.4933622181415558, + "logps/chosen": -427.89312744140625, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -666.23779296875, + "loss": 1.1048, + "margin_dpo/margin_mean": 204.62982177734375, + "margin_dpo/margin_std": 249.5997314453125, + "step": 384 + }, + { + "KL/chosen_KL_mean": -345.30267333984375, + "KL/mean": -467.48583984375, + "KL/rejected_KL_mean": -589.6690063476562, + "KL/std": 224.0086669921875, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0016768588684499264, + "fcm_dpo/delta": -0.016689352691173553, + "fcm_dpo/margin": 244.36630249023438, + "fcm_dpo/q_t": 0.40174418687820435, + "grad_norm": 26.779409408569336, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.5289189219474792, + "logits/rejected": -0.500619649887085, + "logps/chosen": -410.8548583984375, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -667.4969482421875, + "loss": 1.0615, + "margin_dpo/margin_mean": 244.36631774902344, + "margin_dpo/margin_std": 253.60830688476562, + "step": 385 + }, + { + "KL/chosen_KL_mean": -361.995361328125, + "KL/mean": -493.2794494628906, + "KL/rejected_KL_mean": -624.5635375976562, + "KL/std": 274.184326171875, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0015988796949386597, + "fcm_dpo/delta": -0.034051790833473206, + "fcm_dpo/margin": 262.5681457519531, + "fcm_dpo/q_t": 0.400490939617157, + "grad_norm": 25.8364200592041, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.5255472660064697, + "logits/rejected": -0.5233632922172546, + "logps/chosen": -420.21722412109375, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -716.8909912109375, + "loss": 1.0737, + "margin_dpo/margin_mean": 262.56817626953125, + "margin_dpo/margin_std": 327.55181884765625, + "step": 386 + }, + { + "KL/chosen_KL_mean": -404.6907958984375, + "KL/mean": -515.4647827148438, + "KL/rejected_KL_mean": -626.23876953125, + "KL/std": 265.8984375, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0016458687605336308, + "fcm_dpo/delta": 0.05550061911344528, + "fcm_dpo/margin": 221.54798889160156, + "fcm_dpo/q_t": 0.4187946617603302, + "grad_norm": 29.908674240112305, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.5720622539520264, + "logits/rejected": -0.5503807067871094, + "logps/chosen": -471.1102600097656, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -718.407958984375, + "loss": 1.1074, + "margin_dpo/margin_mean": 221.54798889160156, + "margin_dpo/margin_std": 279.3563232421875, + "step": 387 + }, + { + "KL/chosen_KL_mean": -380.31915283203125, + "KL/mean": -527.940185546875, + "KL/rejected_KL_mean": -675.5611572265625, + "KL/std": 304.70587158203125, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.001483785454183817, + "fcm_dpo/delta": -0.07916902750730515, + "fcm_dpo/margin": 295.241943359375, + "fcm_dpo/q_t": 0.39270198345184326, + "grad_norm": 27.048213958740234, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.4747141897678375, + "logits/rejected": -0.4871397018432617, + "logps/chosen": -430.4486083984375, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -779.9942016601562, + "loss": 1.0573, + "margin_dpo/margin_mean": 295.2419738769531, + "margin_dpo/margin_std": 350.07147216796875, + "step": 388 + }, + { + "KL/chosen_KL_mean": -417.1252136230469, + "KL/mean": -543.790771484375, + "KL/rejected_KL_mean": -670.4564208984375, + "KL/std": 307.47637939453125, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.001513138646259904, + "fcm_dpo/delta": 0.026595965027809143, + "fcm_dpo/margin": 253.3311767578125, + "fcm_dpo/q_t": 0.41341066360473633, + "grad_norm": 33.48180389404297, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.5469924211502075, + "logits/rejected": -0.5381365418434143, + "logps/chosen": -475.03179931640625, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -748.3709716796875, + "loss": 1.1045, + "margin_dpo/margin_mean": 253.3311767578125, + "margin_dpo/margin_std": 345.8287353515625, + "step": 389 + }, + { + "KL/chosen_KL_mean": -417.957763671875, + "KL/mean": -541.5369262695312, + "KL/rejected_KL_mean": -665.1160888671875, + "KL/std": 302.0540771484375, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.0015666976105421782, + "fcm_dpo/delta": 0.019385023042559624, + "fcm_dpo/margin": 247.1583251953125, + "fcm_dpo/q_t": 0.4156228303909302, + "grad_norm": 23.625947952270508, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.46019166707992554, + "logits/rejected": -0.45252659916877747, + "logps/chosen": -467.1836853027344, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -750.644287109375, + "loss": 1.1168, + "margin_dpo/margin_mean": 247.1583251953125, + "margin_dpo/margin_std": 383.4603271484375, + "step": 390 + }, + { + "KL/chosen_KL_mean": -418.42572021484375, + "KL/mean": -490.8452453613281, + "KL/rejected_KL_mean": -563.2647705078125, + "KL/std": 288.76385498046875, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0020397088956087828, + "fcm_dpo/delta": 0.14491389691829681, + "fcm_dpo/margin": 144.83901977539062, + "fcm_dpo/q_t": 0.4414842128753662, + "grad_norm": 52.0184440612793, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.472456693649292, + "logits/rejected": -0.4619515538215637, + "logps/chosen": -482.75537109375, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -650.0029296875, + "loss": 1.2088, + "margin_dpo/margin_mean": 144.83901977539062, + "margin_dpo/margin_std": 298.0270690917969, + "step": 391 + }, + { + "KL/chosen_KL_mean": -343.19525146484375, + "KL/mean": -493.396484375, + "KL/rejected_KL_mean": -643.5977783203125, + "KL/std": 300.6968994140625, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0016628424637019634, + "fcm_dpo/delta": -0.20023079216480255, + "fcm_dpo/margin": 300.4024658203125, + "fcm_dpo/q_t": 0.36630189418792725, + "grad_norm": 26.650352478027344, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.4742690920829773, + "logits/rejected": -0.46840304136276245, + "logps/chosen": -396.69921875, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -745.943603515625, + "loss": 1.0092, + "margin_dpo/margin_mean": 300.4024658203125, + "margin_dpo/margin_std": 297.166748046875, + "step": 392 + }, + { + "KL/chosen_KL_mean": -344.81060791015625, + "KL/mean": -455.6504821777344, + "KL/rejected_KL_mean": -566.4903564453125, + "KL/std": 291.33221435546875, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0016052678693085909, + "fcm_dpo/delta": 0.06851550936698914, + "fcm_dpo/margin": 221.67974853515625, + "fcm_dpo/q_t": 0.4255250096321106, + "grad_norm": 21.591970443725586, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.4540834426879883, + "logits/rejected": -0.451620489358902, + "logps/chosen": -391.2845458984375, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -638.459228515625, + "loss": 1.1418, + "margin_dpo/margin_mean": 221.67974853515625, + "margin_dpo/margin_std": 372.8785400390625, + "step": 393 + }, + { + "KL/chosen_KL_mean": -376.7562561035156, + "KL/mean": -496.7586669921875, + "KL/rejected_KL_mean": -616.7610473632812, + "KL/std": 303.6330261230469, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.001655565807595849, + "fcm_dpo/delta": 0.003836357966065407, + "fcm_dpo/margin": 240.00477600097656, + "fcm_dpo/q_t": 0.4095136225223541, + "grad_norm": 21.343175888061523, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.46841922402381897, + "logits/rejected": -0.4693748354911804, + "logps/chosen": -429.6678161621094, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -707.5836791992188, + "loss": 1.0918, + "margin_dpo/margin_mean": 240.00477600097656, + "margin_dpo/margin_std": 317.42437744140625, + "step": 394 + }, + { + "KL/chosen_KL_mean": -362.38995361328125, + "KL/mean": -493.2774658203125, + "KL/rejected_KL_mean": -624.1649780273438, + "KL/std": 293.3901672363281, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0015726612182334065, + "fcm_dpo/delta": -0.024214228615164757, + "fcm_dpo/margin": 261.7750244140625, + "fcm_dpo/q_t": 0.4036139249801636, + "grad_norm": 21.289175033569336, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.5199534893035889, + "logits/rejected": -0.5158529281616211, + "logps/chosen": -424.9360656738281, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -707.9476318359375, + "loss": 1.0849, + "margin_dpo/margin_mean": 261.7750244140625, + "margin_dpo/margin_std": 343.44354248046875, + "step": 395 + }, + { + "KL/chosen_KL_mean": -373.7277526855469, + "KL/mean": -489.446533203125, + "KL/rejected_KL_mean": -605.1654052734375, + "KL/std": 288.4603576660156, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.0016512691508978605, + "fcm_dpo/delta": 0.028826599940657616, + "fcm_dpo/margin": 231.4375457763672, + "fcm_dpo/q_t": 0.41250765323638916, + "grad_norm": 22.668109893798828, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.4854172468185425, + "logits/rejected": -0.4645005464553833, + "logps/chosen": -442.72369384765625, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -693.81201171875, + "loss": 1.0875, + "margin_dpo/margin_mean": 231.43756103515625, + "margin_dpo/margin_std": 270.1766662597656, + "step": 396 + }, + { + "KL/chosen_KL_mean": -345.4547119140625, + "KL/mean": -491.2574462890625, + "KL/rejected_KL_mean": -637.0601806640625, + "KL/std": 283.0622253417969, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0014739397447556257, + "fcm_dpo/delta": -0.06422150880098343, + "fcm_dpo/margin": 291.6054382324219, + "fcm_dpo/q_t": 0.39274823665618896, + "grad_norm": 29.06831169128418, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.46980020403862, + "logits/rejected": -0.47478750348091125, + "logps/chosen": -406.73187255859375, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -740.17626953125, + "loss": 1.0495, + "margin_dpo/margin_mean": 291.6054382324219, + "margin_dpo/margin_std": 302.38970947265625, + "step": 397 + }, + { + "KL/chosen_KL_mean": -366.6300354003906, + "KL/mean": -499.76983642578125, + "KL/rejected_KL_mean": -632.90966796875, + "KL/std": 293.9355773925781, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0014921356923878193, + "fcm_dpo/delta": 0.004290161654353142, + "fcm_dpo/margin": 266.2795715332031, + "fcm_dpo/q_t": 0.40941908955574036, + "grad_norm": 20.657440185546875, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.5278111696243286, + "logits/rejected": -0.5256218314170837, + "logps/chosen": -434.7815856933594, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -741.4332275390625, + "loss": 1.0907, + "margin_dpo/margin_mean": 266.2795715332031, + "margin_dpo/margin_std": 350.179443359375, + "step": 398 + }, + { + "KL/chosen_KL_mean": -315.19482421875, + "KL/mean": -432.66326904296875, + "KL/rejected_KL_mean": -550.1317138671875, + "KL/std": 257.30950927734375, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0015734569169580936, + "fcm_dpo/delta": 0.047637663781642914, + "fcm_dpo/margin": 234.9368133544922, + "fcm_dpo/q_t": 0.41743379831314087, + "grad_norm": 21.86760711669922, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.4569128751754761, + "logits/rejected": -0.41413915157318115, + "logps/chosen": -376.0846252441406, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -628.0972900390625, + "loss": 1.1148, + "margin_dpo/margin_mean": 234.93682861328125, + "margin_dpo/margin_std": 323.26910400390625, + "step": 399 + }, + { + "KL/chosen_KL_mean": -298.4742431640625, + "KL/mean": -467.8236389160156, + "KL/rejected_KL_mean": -637.1730346679688, + "KL/std": 272.68536376953125, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.00134001555852592, + "fcm_dpo/delta": -0.11473602056503296, + "fcm_dpo/margin": 338.6988220214844, + "fcm_dpo/q_t": 0.37980222702026367, + "grad_norm": 19.321056365966797, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.45275765657424927, + "logits/rejected": -0.4457949995994568, + "logps/chosen": -362.1178283691406, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -742.4257202148438, + "loss": 1.0206, + "margin_dpo/margin_mean": 338.6988220214844, + "margin_dpo/margin_std": 297.51031494140625, + "step": 400 + }, + { + "KL/chosen_KL_mean": -347.76898193359375, + "KL/mean": -442.2925109863281, + "KL/rejected_KL_mean": -536.8160400390625, + "KL/std": 280.5494384765625, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.00160554563626647, + "fcm_dpo/delta": 0.13878443837165833, + "fcm_dpo/margin": 189.04698181152344, + "fcm_dpo/q_t": 0.43937820196151733, + "grad_norm": 21.756427764892578, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.4988451600074768, + "logits/rejected": -0.492392897605896, + "logps/chosen": -404.9320373535156, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -620.6085205078125, + "loss": 1.1925, + "margin_dpo/margin_mean": 189.0469970703125, + "margin_dpo/margin_std": 374.1167297363281, + "step": 401 + }, + { + "KL/chosen_KL_mean": -258.14959716796875, + "KL/mean": -438.02703857421875, + "KL/rejected_KL_mean": -617.9044799804688, + "KL/std": 310.0885009765625, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0013267784379422665, + "fcm_dpo/delta": -0.1641661822795868, + "fcm_dpo/margin": 359.75494384765625, + "fcm_dpo/q_t": 0.37116238474845886, + "grad_norm": 20.190622329711914, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.4920314848423004, + "logits/rejected": -0.480332612991333, + "logps/chosen": -308.88995361328125, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -698.9505615234375, + "loss": 1.0061, + "margin_dpo/margin_mean": 359.75494384765625, + "margin_dpo/margin_std": 319.1515808105469, + "step": 402 + }, + { + "KL/chosen_KL_mean": -292.50390625, + "KL/mean": -428.873291015625, + "KL/rejected_KL_mean": -565.24267578125, + "KL/std": 291.69244384765625, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0013493448495864868, + "fcm_dpo/delta": 0.050415799021720886, + "fcm_dpo/margin": 272.73876953125, + "fcm_dpo/q_t": 0.4170788526535034, + "grad_norm": 22.705007553100586, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.4615529775619507, + "logits/rejected": -0.4557945728302002, + "logps/chosen": -339.6512145996094, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -642.50927734375, + "loss": 1.0941, + "margin_dpo/margin_mean": 272.73876953125, + "margin_dpo/margin_std": 306.66265869140625, + "step": 403 + }, + { + "KL/chosen_KL_mean": -336.2072448730469, + "KL/mean": -466.8104248046875, + "KL/rejected_KL_mean": -597.41357421875, + "KL/std": 278.45245361328125, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.001444081193767488, + "fcm_dpo/delta": 0.03668762370944023, + "fcm_dpo/margin": 261.20635986328125, + "fcm_dpo/q_t": 0.41537854075431824, + "grad_norm": 29.176677703857422, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.45944684743881226, + "logits/rejected": -0.4487238824367523, + "logps/chosen": -384.08251953125, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -674.568603515625, + "loss": 1.1104, + "margin_dpo/margin_mean": 261.20635986328125, + "margin_dpo/margin_std": 361.2408447265625, + "step": 404 + }, + { + "KL/chosen_KL_mean": -388.7740478515625, + "KL/mean": -496.8353271484375, + "KL/rejected_KL_mean": -604.8966064453125, + "KL/std": 312.28668212890625, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.0014538425020873547, + "fcm_dpo/delta": -0.026516491547226906, + "fcm_dpo/margin": 216.12255859375, + "fcm_dpo/q_t": 0.42757102847099304, + "grad_norm": 29.22574234008789, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.5201287865638733, + "logits/rejected": -0.5093380212783813, + "logps/chosen": -453.9369812011719, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -692.0834350585938, + "loss": 1.1806, + "margin_dpo/margin_mean": 216.12255859375, + "margin_dpo/margin_std": 396.87908935546875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -355.637451171875, + "KL/mean": -501.5604248046875, + "KL/rejected_KL_mean": -647.4833984375, + "KL/std": 317.0197448730469, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0014266908401623368, + "fcm_dpo/delta": -0.033352650701999664, + "fcm_dpo/margin": 291.8459167480469, + "fcm_dpo/q_t": 0.4017193913459778, + "grad_norm": 21.75078773498535, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.5137743949890137, + "logits/rejected": -0.5227109789848328, + "logps/chosen": -405.3782958984375, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -739.56201171875, + "loss": 1.0792, + "margin_dpo/margin_mean": 291.8459167480469, + "margin_dpo/margin_std": 360.86712646484375, + "step": 406 + }, + { + "KL/chosen_KL_mean": -375.3631896972656, + "KL/mean": -470.50189208984375, + "KL/rejected_KL_mean": -565.6405639648438, + "KL/std": 258.78857421875, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0013535934267565608, + "fcm_dpo/delta": 0.02188667096197605, + "fcm_dpo/margin": 190.27740478515625, + "fcm_dpo/q_t": 0.43996331095695496, + "grad_norm": 23.94710350036621, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.5585223436355591, + "logits/rejected": -0.5322338938713074, + "logps/chosen": -431.69390869140625, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -643.1527099609375, + "loss": 1.1953, + "margin_dpo/margin_mean": 190.2774200439453, + "margin_dpo/margin_std": 333.7882385253906, + "step": 407 + }, + { + "KL/chosen_KL_mean": -389.05035400390625, + "KL/mean": -490.5750427246094, + "KL/rejected_KL_mean": -592.0997314453125, + "KL/std": 244.0631866455078, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.0016291262581944466, + "fcm_dpo/delta": 0.09969305992126465, + "fcm_dpo/margin": 203.04937744140625, + "fcm_dpo/q_t": 0.4286070466041565, + "grad_norm": 28.127901077270508, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.5659944415092468, + "logits/rejected": -0.5408339500427246, + "logps/chosen": -458.83966064453125, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -682.1966552734375, + "loss": 1.1308, + "margin_dpo/margin_mean": 203.04937744140625, + "margin_dpo/margin_std": 274.0846862792969, + "step": 408 + }, + { + "KL/chosen_KL_mean": -381.6026306152344, + "KL/mean": -485.71112060546875, + "KL/rejected_KL_mean": -589.819580078125, + "KL/std": 286.2164306640625, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0018248842097818851, + "fcm_dpo/delta": 0.02699420042335987, + "fcm_dpo/margin": 208.21694946289062, + "fcm_dpo/q_t": 0.4149779677391052, + "grad_norm": 42.70933532714844, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.4948524832725525, + "logits/rejected": -0.47649848461151123, + "logps/chosen": -448.9200744628906, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -674.723876953125, + "loss": 1.1244, + "margin_dpo/margin_mean": 208.21694946289062, + "margin_dpo/margin_std": 319.69189453125, + "step": 409 + }, + { + "KL/chosen_KL_mean": -351.9015808105469, + "KL/mean": -478.4359130859375, + "KL/rejected_KL_mean": -604.9702758789062, + "KL/std": 285.3361511230469, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0016538414638489485, + "fcm_dpo/delta": -0.03480095416307449, + "fcm_dpo/margin": 253.06863403320312, + "fcm_dpo/q_t": 0.3992757201194763, + "grad_norm": 35.037715911865234, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.5824193954467773, + "logits/rejected": -0.5856485366821289, + "logps/chosen": -403.366943359375, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -688.169189453125, + "loss": 1.079, + "margin_dpo/margin_mean": 253.06863403320312, + "margin_dpo/margin_std": 324.7110595703125, + "step": 410 + }, + { + "KL/chosen_KL_mean": -374.0163269042969, + "KL/mean": -491.8541259765625, + "KL/rejected_KL_mean": -609.69189453125, + "KL/std": 309.1568603515625, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0014088283060118556, + "fcm_dpo/delta": -0.08271211385726929, + "fcm_dpo/margin": 235.67556762695312, + "fcm_dpo/q_t": 0.4166325628757477, + "grad_norm": 32.5038948059082, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.5615779757499695, + "logits/rejected": -0.566271185874939, + "logps/chosen": -426.3236083984375, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -690.3868408203125, + "loss": 1.1475, + "margin_dpo/margin_mean": 235.67556762695312, + "margin_dpo/margin_std": 364.743408203125, + "step": 411 + }, + { + "KL/chosen_KL_mean": -376.010009765625, + "KL/mean": -504.2699279785156, + "KL/rejected_KL_mean": -632.5298461914062, + "KL/std": 297.99005126953125, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0014877468347549438, + "fcm_dpo/delta": 0.028690597042441368, + "fcm_dpo/margin": 256.51983642578125, + "fcm_dpo/q_t": 0.4138166904449463, + "grad_norm": 27.262081146240234, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.6011987924575806, + "logits/rejected": -0.6305391788482666, + "logps/chosen": -429.1541442871094, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -732.5906372070312, + "loss": 1.1073, + "margin_dpo/margin_mean": 256.51983642578125, + "margin_dpo/margin_std": 358.5880126953125, + "step": 412 + }, + { + "KL/chosen_KL_mean": -413.91949462890625, + "KL/mean": -544.1796264648438, + "KL/rejected_KL_mean": -674.4398193359375, + "KL/std": 308.14947509765625, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0015282519161701202, + "fcm_dpo/delta": 0.0018488653004169464, + "fcm_dpo/margin": 260.52032470703125, + "fcm_dpo/q_t": 0.4066236615180969, + "grad_norm": 31.31689453125, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.5308142304420471, + "logits/rejected": -0.5381065607070923, + "logps/chosen": -475.5014343261719, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -773.9132080078125, + "loss": 1.0953, + "margin_dpo/margin_mean": 260.52032470703125, + "margin_dpo/margin_std": 349.33416748046875, + "step": 413 + }, + { + "KL/chosen_KL_mean": -376.8102722167969, + "KL/mean": -513.2676391601562, + "KL/rejected_KL_mean": -649.7249755859375, + "KL/std": 290.8931884765625, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0015321837272495031, + "fcm_dpo/delta": -0.043847713619470596, + "fcm_dpo/margin": 272.91473388671875, + "fcm_dpo/q_t": 0.3983882963657379, + "grad_norm": 32.68032455444336, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.5271749496459961, + "logits/rejected": -0.5175067186355591, + "logps/chosen": -423.4417724609375, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -737.3715209960938, + "loss": 1.0877, + "margin_dpo/margin_mean": 272.91473388671875, + "margin_dpo/margin_std": 341.33642578125, + "step": 414 + }, + { + "KL/chosen_KL_mean": -399.9847412109375, + "KL/mean": -505.3280029296875, + "KL/rejected_KL_mean": -610.6712646484375, + "KL/std": 276.31341552734375, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0015684034442529082, + "fcm_dpo/delta": 0.10484690964221954, + "fcm_dpo/margin": 210.68653869628906, + "fcm_dpo/q_t": 0.42983919382095337, + "grad_norm": 25.915630340576172, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.5725701451301575, + "logits/rejected": -0.567946195602417, + "logps/chosen": -478.60302734375, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -711.1488037109375, + "loss": 1.144, + "margin_dpo/margin_mean": 210.68653869628906, + "margin_dpo/margin_std": 311.3158264160156, + "step": 415 + }, + { + "KL/chosen_KL_mean": -365.5072021484375, + "KL/mean": -527.9451904296875, + "KL/rejected_KL_mean": -690.3831787109375, + "KL/std": 313.2249755859375, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.0014323981013149023, + "fcm_dpo/delta": -0.1226118803024292, + "fcm_dpo/margin": 324.8759460449219, + "fcm_dpo/q_t": 0.37992632389068604, + "grad_norm": 27.974157333374023, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.5598719120025635, + "logits/rejected": -0.564556896686554, + "logps/chosen": -423.7863464355469, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -780.951904296875, + "loss": 1.024, + "margin_dpo/margin_mean": 324.8759460449219, + "margin_dpo/margin_std": 312.55938720703125, + "step": 416 + }, + { + "KL/chosen_KL_mean": -353.57537841796875, + "KL/mean": -489.65478515625, + "KL/rejected_KL_mean": -625.7341918945312, + "KL/std": 285.030029296875, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.0013993415050208569, + "fcm_dpo/delta": 0.030016159638762474, + "fcm_dpo/margin": 272.1588134765625, + "fcm_dpo/q_t": 0.4119567573070526, + "grad_norm": 29.711999893188477, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.5493444204330444, + "logits/rejected": -0.5241958498954773, + "logps/chosen": -403.7740783691406, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -693.8860473632812, + "loss": 1.0801, + "margin_dpo/margin_mean": 272.1588439941406, + "margin_dpo/margin_std": 288.48846435546875, + "step": 417 + }, + { + "KL/chosen_KL_mean": -389.98394775390625, + "KL/mean": -531.188232421875, + "KL/rejected_KL_mean": -672.392578125, + "KL/std": 321.02679443359375, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0014645163901150227, + "fcm_dpo/delta": -0.04145345836877823, + "fcm_dpo/margin": 282.40869140625, + "fcm_dpo/q_t": 0.4018082022666931, + "grad_norm": 22.506240844726562, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.6127409338951111, + "logits/rejected": -0.5916212201118469, + "logps/chosen": -471.96240234375, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -777.0841064453125, + "loss": 1.1022, + "margin_dpo/margin_mean": 282.40869140625, + "margin_dpo/margin_std": 376.09161376953125, + "step": 418 + }, + { + "KL/chosen_KL_mean": -354.58636474609375, + "KL/mean": -506.98297119140625, + "KL/rejected_KL_mean": -659.379638671875, + "KL/std": 276.9751281738281, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0013055275194346905, + "fcm_dpo/delta": 0.0034003122709691525, + "fcm_dpo/margin": 304.79327392578125, + "fcm_dpo/q_t": 0.4058445692062378, + "grad_norm": 25.18451499938965, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.5361112356185913, + "logits/rejected": -0.5325924158096313, + "logps/chosen": -407.5350341796875, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -750.9627685546875, + "loss": 1.0692, + "margin_dpo/margin_mean": 304.79327392578125, + "margin_dpo/margin_std": 320.16162109375, + "step": 419 + }, + { + "KL/chosen_KL_mean": -448.81329345703125, + "KL/mean": -544.2679443359375, + "KL/rejected_KL_mean": -639.7225341796875, + "KL/std": 302.1771240234375, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.001609490136615932, + "fcm_dpo/delta": 0.12760579586029053, + "fcm_dpo/margin": 190.9093475341797, + "fcm_dpo/q_t": 0.43786126375198364, + "grad_norm": 57.40446090698242, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.5600706338882446, + "logits/rejected": -0.5160965919494629, + "logps/chosen": -526.5831909179688, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -709.04248046875, + "loss": 1.2016, + "margin_dpo/margin_mean": 190.9093475341797, + "margin_dpo/margin_std": 403.58026123046875, + "step": 420 + }, + { + "KL/chosen_KL_mean": -383.8134765625, + "KL/mean": -487.9678955078125, + "KL/rejected_KL_mean": -592.122314453125, + "KL/std": 293.6025390625, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0018344377167522907, + "fcm_dpo/delta": 0.018156960606575012, + "fcm_dpo/margin": 208.3088836669922, + "fcm_dpo/q_t": 0.4109729826450348, + "grad_norm": 32.440040588378906, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.6202581524848938, + "logits/rejected": -0.6075379848480225, + "logps/chosen": -437.579345703125, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -681.40380859375, + "loss": 1.1322, + "margin_dpo/margin_mean": 208.3088836669922, + "margin_dpo/margin_std": 319.2204895019531, + "step": 421 + }, + { + "KL/chosen_KL_mean": -423.02252197265625, + "KL/mean": -545.48046875, + "KL/rejected_KL_mean": -667.9382934570312, + "KL/std": 296.6886901855469, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0016887241508811712, + "fcm_dpo/delta": -0.023193474858999252, + "fcm_dpo/margin": 244.91578674316406, + "fcm_dpo/q_t": 0.40264299511909485, + "grad_norm": 44.55949783325195, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.5801171064376831, + "logits/rejected": -0.5803545713424683, + "logps/chosen": -491.65631103515625, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -755.8018188476562, + "loss": 1.0928, + "margin_dpo/margin_mean": 244.91578674316406, + "margin_dpo/margin_std": 343.1561584472656, + "step": 422 + }, + { + "KL/chosen_KL_mean": -408.7502136230469, + "KL/mean": -549.06884765625, + "KL/rejected_KL_mean": -689.3875122070312, + "KL/std": 292.96270751953125, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0015233319718390703, + "fcm_dpo/delta": -0.05018993839621544, + "fcm_dpo/margin": 280.6373291015625, + "fcm_dpo/q_t": 0.3961368799209595, + "grad_norm": 26.388099670410156, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.5856159329414368, + "logits/rejected": -0.5923899412155151, + "logps/chosen": -463.7405090332031, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -775.694091796875, + "loss": 1.0592, + "margin_dpo/margin_mean": 280.6373596191406, + "margin_dpo/margin_std": 320.4229431152344, + "step": 423 + }, + { + "KL/chosen_KL_mean": -368.3785400390625, + "KL/mean": -496.5576477050781, + "KL/rejected_KL_mean": -624.7366943359375, + "KL/std": 285.81842041015625, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0015009802300482988, + "fcm_dpo/delta": 0.022251751273870468, + "fcm_dpo/margin": 256.358154296875, + "fcm_dpo/q_t": 0.4135265052318573, + "grad_norm": 33.62800598144531, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.6009132266044617, + "logits/rejected": -0.5754563808441162, + "logps/chosen": -424.39044189453125, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -691.2156982421875, + "loss": 1.106, + "margin_dpo/margin_mean": 256.358154296875, + "margin_dpo/margin_std": 352.6119384765625, + "step": 424 + }, + { + "KL/chosen_KL_mean": -417.15570068359375, + "KL/mean": -515.7978515625, + "KL/rejected_KL_mean": -614.43994140625, + "KL/std": 278.95025634765625, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.0017970511689782143, + "fcm_dpo/delta": 0.062237467616796494, + "fcm_dpo/margin": 197.28427124023438, + "fcm_dpo/q_t": 0.4205625057220459, + "grad_norm": 29.63746452331543, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.5688070058822632, + "logits/rejected": -0.5690561532974243, + "logps/chosen": -464.024658203125, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -710.3654174804688, + "loss": 1.1556, + "margin_dpo/margin_mean": 197.28427124023438, + "margin_dpo/margin_std": 347.9555358886719, + "step": 425 + }, + { + "KL/chosen_KL_mean": -396.1029052734375, + "KL/mean": -517.9900512695312, + "KL/rejected_KL_mean": -639.877197265625, + "KL/std": 274.98834228515625, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0017517898231744766, + "fcm_dpo/delta": -0.0511149987578392, + "fcm_dpo/margin": 243.7742919921875, + "fcm_dpo/q_t": 0.3971126079559326, + "grad_norm": 32.52616882324219, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.5753895044326782, + "logits/rejected": -0.5384079813957214, + "logps/chosen": -472.6864318847656, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -721.143798828125, + "loss": 1.0773, + "margin_dpo/margin_mean": 243.7742919921875, + "margin_dpo/margin_std": 307.3427734375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -360.6373596191406, + "KL/mean": -457.64605712890625, + "KL/rejected_KL_mean": -554.6547241210938, + "KL/std": 240.3179473876953, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.001793712843209505, + "fcm_dpo/delta": 0.08003441989421844, + "fcm_dpo/margin": 194.01736450195312, + "fcm_dpo/q_t": 0.4255453646183014, + "grad_norm": 28.326711654663086, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.6111181974411011, + "logits/rejected": -0.5922002792358398, + "logps/chosen": -425.4912414550781, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -633.220703125, + "loss": 1.1332, + "margin_dpo/margin_mean": 194.01736450195312, + "margin_dpo/margin_std": 291.666015625, + "step": 427 + }, + { + "KL/chosen_KL_mean": -443.464111328125, + "KL/mean": -577.4393310546875, + "KL/rejected_KL_mean": -711.41455078125, + "KL/std": 321.87640380859375, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.001614520326256752, + "fcm_dpo/delta": -0.07145649194717407, + "fcm_dpo/margin": 267.9504089355469, + "fcm_dpo/q_t": 0.394869327545166, + "grad_norm": 29.2352294921875, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.5986416339874268, + "logits/rejected": -0.6018632650375366, + "logps/chosen": -506.10076904296875, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -814.6963500976562, + "loss": 1.0951, + "margin_dpo/margin_mean": 267.9504089355469, + "margin_dpo/margin_std": 396.5560302734375, + "step": 428 + }, + { + "KL/chosen_KL_mean": -453.14031982421875, + "KL/mean": -558.6182861328125, + "KL/rejected_KL_mean": -664.09619140625, + "KL/std": 281.262939453125, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.0015741197858005762, + "fcm_dpo/delta": -0.040317293256521225, + "fcm_dpo/margin": 210.95590209960938, + "fcm_dpo/q_t": 0.4206579625606537, + "grad_norm": 29.22441864013672, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.6086622476577759, + "logits/rejected": -0.581855297088623, + "logps/chosen": -534.3743286132812, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -755.89111328125, + "loss": 1.1602, + "margin_dpo/margin_mean": 210.95590209960938, + "margin_dpo/margin_std": 341.42901611328125, + "step": 429 + }, + { + "KL/chosen_KL_mean": -381.8338623046875, + "KL/mean": -485.0985107421875, + "KL/rejected_KL_mean": -588.3631591796875, + "KL/std": 265.72589111328125, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.001708697760477662, + "fcm_dpo/delta": 0.06908033788204193, + "fcm_dpo/margin": 206.52932739257812, + "fcm_dpo/q_t": 0.4227146804332733, + "grad_norm": 40.607749938964844, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.5892158150672913, + "logits/rejected": -0.5879628658294678, + "logps/chosen": -442.754150390625, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -692.7860107421875, + "loss": 1.1248, + "margin_dpo/margin_mean": 206.52932739257812, + "margin_dpo/margin_std": 297.7901611328125, + "step": 430 + }, + { + "KL/chosen_KL_mean": -354.79736328125, + "KL/mean": -504.74810791015625, + "KL/rejected_KL_mean": -654.6988525390625, + "KL/std": 294.78875732421875, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0014870925806462765, + "fcm_dpo/delta": -0.0890204980969429, + "fcm_dpo/margin": 299.9014892578125, + "fcm_dpo/q_t": 0.3885713517665863, + "grad_norm": 27.04310417175293, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.6147454977035522, + "logits/rejected": -0.6116945743560791, + "logps/chosen": -412.1461181640625, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -747.5390625, + "loss": 1.0492, + "margin_dpo/margin_mean": 299.9014892578125, + "margin_dpo/margin_std": 333.67730712890625, + "step": 431 + }, + { + "KL/chosen_KL_mean": -351.54315185546875, + "KL/mean": -501.9591979980469, + "KL/rejected_KL_mean": -652.3753051757812, + "KL/std": 292.12811279296875, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.0014298271853476763, + "fcm_dpo/delta": -0.05924157053232193, + "fcm_dpo/margin": 300.8321533203125, + "fcm_dpo/q_t": 0.39376306533813477, + "grad_norm": 31.670360565185547, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.6010767221450806, + "logits/rejected": -0.5897285342216492, + "logps/chosen": -422.617919921875, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -750.954833984375, + "loss": 1.0523, + "margin_dpo/margin_mean": 300.8321533203125, + "margin_dpo/margin_std": 296.0356140136719, + "step": 432 + }, + { + "KL/chosen_KL_mean": -445.81146240234375, + "KL/mean": -543.8163452148438, + "KL/rejected_KL_mean": -641.8211669921875, + "KL/std": 302.93316650390625, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0015859071863815188, + "fcm_dpo/delta": 0.12226778268814087, + "fcm_dpo/margin": 196.00973510742188, + "fcm_dpo/q_t": 0.43242147564888, + "grad_norm": 38.35377502441406, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.6371247172355652, + "logits/rejected": -0.6237683892250061, + "logps/chosen": -504.08465576171875, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -737.7720947265625, + "loss": 1.1783, + "margin_dpo/margin_mean": 196.00973510742188, + "margin_dpo/margin_std": 365.50811767578125, + "step": 433 + }, + { + "KL/chosen_KL_mean": -385.5121765136719, + "KL/mean": -504.4186096191406, + "KL/rejected_KL_mean": -623.3250122070312, + "KL/std": 285.87884521484375, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0016222589183598757, + "fcm_dpo/delta": 0.019110068678855896, + "fcm_dpo/margin": 237.81283569335938, + "fcm_dpo/q_t": 0.4163554310798645, + "grad_norm": 31.304502487182617, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.6609657406806946, + "logits/rejected": -0.6543369889259338, + "logps/chosen": -447.48590087890625, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -701.8236083984375, + "loss": 1.113, + "margin_dpo/margin_mean": 237.8128204345703, + "margin_dpo/margin_std": 357.0226745605469, + "step": 434 + }, + { + "KL/chosen_KL_mean": -366.02606201171875, + "KL/mean": -493.39617919921875, + "KL/rejected_KL_mean": -620.766357421875, + "KL/std": 283.81414794921875, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.001610426465049386, + "fcm_dpo/delta": -0.020799141377210617, + "fcm_dpo/margin": 254.740234375, + "fcm_dpo/q_t": 0.4033718705177307, + "grad_norm": 30.6375789642334, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.6943074464797974, + "logits/rejected": -0.6805083751678467, + "logps/chosen": -417.5281066894531, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -708.333251953125, + "loss": 1.0828, + "margin_dpo/margin_mean": 254.74024963378906, + "margin_dpo/margin_std": 321.0318603515625, + "step": 435 + }, + { + "KL/chosen_KL_mean": -373.72186279296875, + "KL/mean": -489.437744140625, + "KL/rejected_KL_mean": -605.153564453125, + "KL/std": 257.1832580566406, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0016507278196513653, + "fcm_dpo/delta": 0.02712010033428669, + "fcm_dpo/margin": 231.43165588378906, + "fcm_dpo/q_t": 0.41315099596977234, + "grad_norm": 30.587175369262695, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.6558730602264404, + "logits/rejected": -0.6176996827125549, + "logps/chosen": -445.1256103515625, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -687.88134765625, + "loss": 1.103, + "margin_dpo/margin_mean": 231.43167114257812, + "margin_dpo/margin_std": 304.65887451171875, + "step": 436 + }, + { + "KL/chosen_KL_mean": -369.8938293457031, + "KL/mean": -485.63037109375, + "KL/rejected_KL_mean": -601.366943359375, + "KL/std": 242.96083068847656, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0017139371484518051, + "fcm_dpo/delta": 0.004689780995249748, + "fcm_dpo/margin": 231.47308349609375, + "fcm_dpo/q_t": 0.40768975019454956, + "grad_norm": 25.386877059936523, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.6826403141021729, + "logits/rejected": -0.6562011241912842, + "logps/chosen": -434.6380920410156, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -683.4104614257812, + "loss": 1.0842, + "margin_dpo/margin_mean": 231.47308349609375, + "margin_dpo/margin_std": 282.5240478515625, + "step": 437 + }, + { + "KL/chosen_KL_mean": -385.8797302246094, + "KL/mean": -514.9544677734375, + "KL/rejected_KL_mean": -644.0291748046875, + "KL/std": 284.6157531738281, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0015868828631937504, + "fcm_dpo/delta": -0.027686219662427902, + "fcm_dpo/margin": 258.14947509765625, + "fcm_dpo/q_t": 0.4011117219924927, + "grad_norm": 36.56245803833008, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.6592687368392944, + "logits/rejected": -0.6328068971633911, + "logps/chosen": -444.8984069824219, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -727.1060180664062, + "loss": 1.0781, + "margin_dpo/margin_mean": 258.14947509765625, + "margin_dpo/margin_std": 307.8912658691406, + "step": 438 + }, + { + "KL/chosen_KL_mean": -397.440185546875, + "KL/mean": -502.10455322265625, + "KL/rejected_KL_mean": -606.7689208984375, + "KL/std": 281.3536682128906, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0014228572836145759, + "fcm_dpo/delta": -0.07547041773796082, + "fcm_dpo/margin": 209.32879638671875, + "fcm_dpo/q_t": 0.4233711063861847, + "grad_norm": 26.591917037963867, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.6513150930404663, + "logits/rejected": -0.6519348621368408, + "logps/chosen": -451.2242431640625, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -690.75439453125, + "loss": 1.1686, + "margin_dpo/margin_mean": 209.32876586914062, + "margin_dpo/margin_std": 326.3582763671875, + "step": 439 + }, + { + "KL/chosen_KL_mean": -424.1907043457031, + "KL/mean": -549.408203125, + "KL/rejected_KL_mean": -674.625732421875, + "KL/std": 355.38433837890625, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0015238459454849362, + "fcm_dpo/delta": 0.02612234093248844, + "fcm_dpo/margin": 250.43495178222656, + "fcm_dpo/q_t": 0.41634005308151245, + "grad_norm": 36.65753936767578, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.6994473934173584, + "logits/rejected": -0.6900968551635742, + "logps/chosen": -502.7574462890625, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -771.1234741210938, + "loss": 1.1072, + "margin_dpo/margin_mean": 250.43496704101562, + "margin_dpo/margin_std": 356.3500671386719, + "step": 440 + }, + { + "KL/chosen_KL_mean": -473.6324462890625, + "KL/mean": -599.330322265625, + "KL/rejected_KL_mean": -725.0281982421875, + "KL/std": 360.2265625, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.0015725798439234495, + "fcm_dpo/delta": 0.00260746106505394, + "fcm_dpo/margin": 251.39569091796875, + "fcm_dpo/q_t": 0.41514575481414795, + "grad_norm": 36.14642333984375, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.6896128058433533, + "logits/rejected": -0.6732909679412842, + "logps/chosen": -534.4569091796875, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -821.4990234375, + "loss": 1.1421, + "margin_dpo/margin_mean": 251.39569091796875, + "margin_dpo/margin_std": 446.50982666015625, + "step": 441 + }, + { + "KL/chosen_KL_mean": -402.5406494140625, + "KL/mean": -554.17041015625, + "KL/rejected_KL_mean": -705.8001098632812, + "KL/std": 349.5821533203125, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0014577994588762522, + "fcm_dpo/delta": -0.08352486789226532, + "fcm_dpo/margin": 303.25946044921875, + "fcm_dpo/q_t": 0.3944917619228363, + "grad_norm": 34.06388473510742, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.6561492085456848, + "logits/rejected": -0.6455131769180298, + "logps/chosen": -449.5518798828125, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -782.33935546875, + "loss": 1.0824, + "margin_dpo/margin_mean": 303.2594299316406, + "margin_dpo/margin_std": 401.3167419433594, + "step": 442 + }, + { + "KL/chosen_KL_mean": -470.502197265625, + "KL/mean": -563.5130615234375, + "KL/rejected_KL_mean": -656.52392578125, + "KL/std": 353.9302673339844, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0013765229377895594, + "fcm_dpo/delta": 0.0382419228553772, + "fcm_dpo/margin": 186.021728515625, + "fcm_dpo/q_t": 0.4441724121570587, + "grad_norm": 40.15092086791992, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.6763448119163513, + "logits/rejected": -0.6484371423721313, + "logps/chosen": -541.7752075195312, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -743.203857421875, + "loss": 1.2325, + "margin_dpo/margin_mean": 186.021728515625, + "margin_dpo/margin_std": 431.4246826171875, + "step": 443 + }, + { + "KL/chosen_KL_mean": -461.93658447265625, + "KL/mean": -613.6505126953125, + "KL/rejected_KL_mean": -765.364501953125, + "KL/std": 370.37298583984375, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.0013226409209892154, + "fcm_dpo/delta": -0.005307670682668686, + "fcm_dpo/margin": 303.42791748046875, + "fcm_dpo/q_t": 0.4082014858722687, + "grad_norm": 33.01715087890625, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.6438364386558533, + "logits/rejected": -0.6463167667388916, + "logps/chosen": -519.1502685546875, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -862.619384765625, + "loss": 1.0982, + "margin_dpo/margin_mean": 303.42791748046875, + "margin_dpo/margin_std": 423.27032470703125, + "step": 444 + }, + { + "KL/chosen_KL_mean": -402.2928771972656, + "KL/mean": -540.4663696289062, + "KL/rejected_KL_mean": -678.639892578125, + "KL/std": 285.4811706542969, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0014118524268269539, + "fcm_dpo/delta": 0.015325665473937988, + "fcm_dpo/margin": 276.34698486328125, + "fcm_dpo/q_t": 0.4115728735923767, + "grad_norm": 34.8456916809082, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.6687377691268921, + "logits/rejected": -0.6554244756698608, + "logps/chosen": -469.5926818847656, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -771.322509765625, + "loss": 1.0992, + "margin_dpo/margin_mean": 276.34698486328125, + "margin_dpo/margin_std": 367.69573974609375, + "step": 445 + }, + { + "KL/chosen_KL_mean": -360.5770568847656, + "KL/mean": -518.53955078125, + "KL/rejected_KL_mean": -676.5020751953125, + "KL/std": 312.20379638671875, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.00137224025093019, + "fcm_dpo/delta": -0.06505385041236877, + "fcm_dpo/margin": 315.92498779296875, + "fcm_dpo/q_t": 0.39190033078193665, + "grad_norm": 26.528141021728516, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.6326192617416382, + "logits/rejected": -0.6437954902648926, + "logps/chosen": -419.675537109375, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -777.7662353515625, + "loss": 1.0557, + "margin_dpo/margin_mean": 315.92498779296875, + "margin_dpo/margin_std": 325.07269287109375, + "step": 446 + }, + { + "KL/chosen_KL_mean": -383.163330078125, + "KL/mean": -541.930908203125, + "KL/rejected_KL_mean": -700.698486328125, + "KL/std": 360.7058410644531, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0012335461797192693, + "fcm_dpo/delta": 0.013036707416176796, + "fcm_dpo/margin": 317.53509521484375, + "fcm_dpo/q_t": 0.41153937578201294, + "grad_norm": 25.36223602294922, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.6875368356704712, + "logits/rejected": -0.7032911777496338, + "logps/chosen": -439.2386779785156, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -799.3931884765625, + "loss": 1.0899, + "margin_dpo/margin_mean": 317.5351257324219, + "margin_dpo/margin_std": 410.7313232421875, + "step": 447 + }, + { + "KL/chosen_KL_mean": -426.30487060546875, + "KL/mean": -575.4131469726562, + "KL/rejected_KL_mean": -724.521484375, + "KL/std": 317.2506103515625, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0013242221903055906, + "fcm_dpo/delta": 0.004273921251296997, + "fcm_dpo/margin": 298.21661376953125, + "fcm_dpo/q_t": 0.4076346755027771, + "grad_norm": 29.84330940246582, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.5890108942985535, + "logits/rejected": -0.5925085544586182, + "logps/chosen": -486.3087158203125, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -826.7861328125, + "loss": 1.0881, + "margin_dpo/margin_mean": 298.21661376953125, + "margin_dpo/margin_std": 366.0950927734375, + "step": 448 + }, + { + "KL/chosen_KL_mean": -432.274658203125, + "KL/mean": -588.0142822265625, + "KL/rejected_KL_mean": -743.7538452148438, + "KL/std": 391.2838134765625, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.0012739100493490696, + "fcm_dpo/delta": 0.005134426523000002, + "fcm_dpo/margin": 311.4792175292969, + "fcm_dpo/q_t": 0.4119264483451843, + "grad_norm": 24.36716079711914, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.6395070552825928, + "logits/rejected": -0.6301548480987549, + "logps/chosen": -499.80126953125, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -832.3507690429688, + "loss": 1.1068, + "margin_dpo/margin_mean": 311.4792175292969, + "margin_dpo/margin_std": 466.9681396484375, + "step": 449 + }, + { + "KL/chosen_KL_mean": -350.74542236328125, + "KL/mean": -518.7066040039062, + "KL/rejected_KL_mean": -686.6678466796875, + "KL/std": 333.5655822753906, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0012484046164900064, + "fcm_dpo/delta": -0.03335545212030411, + "fcm_dpo/margin": 335.9223937988281, + "fcm_dpo/q_t": 0.3976227641105652, + "grad_norm": 36.70199203491211, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.6501277089118958, + "logits/rejected": -0.6594793796539307, + "logps/chosen": -407.853515625, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -789.4227294921875, + "loss": 1.052, + "margin_dpo/margin_mean": 335.92236328125, + "margin_dpo/margin_std": 331.643310546875, + "step": 450 + }, + { + "KL/chosen_KL_mean": -478.4472351074219, + "KL/mean": -596.5457763671875, + "KL/rejected_KL_mean": -714.6444091796875, + "KL/std": 388.7025146484375, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0014245854690670967, + "fcm_dpo/delta": 0.08672139793634415, + "fcm_dpo/margin": 236.19717407226562, + "fcm_dpo/q_t": 0.42556342482566833, + "grad_norm": 40.344722747802734, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.6830481290817261, + "logits/rejected": -0.6519405245780945, + "logps/chosen": -536.9160766601562, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -787.5737915039062, + "loss": 1.1783, + "margin_dpo/margin_mean": 236.19717407226562, + "margin_dpo/margin_std": 461.38665771484375, + "step": 451 + }, + { + "KL/chosen_KL_mean": -339.47760009765625, + "KL/mean": -487.0719299316406, + "KL/rejected_KL_mean": -634.666259765625, + "KL/std": 301.2289733886719, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0013748779892921448, + "fcm_dpo/delta": -0.012026989832520485, + "fcm_dpo/margin": 295.1885986328125, + "fcm_dpo/q_t": 0.40248599648475647, + "grad_norm": 21.888635635375977, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.6212879419326782, + "logits/rejected": -0.638474702835083, + "logps/chosen": -389.8681640625, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -732.4376831054688, + "loss": 1.0635, + "margin_dpo/margin_mean": 295.1885986328125, + "margin_dpo/margin_std": 304.22344970703125, + "step": 452 + }, + { + "KL/chosen_KL_mean": -394.113525390625, + "KL/mean": -535.056884765625, + "KL/rejected_KL_mean": -676.0001831054688, + "KL/std": 311.166259765625, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0013874982250854373, + "fcm_dpo/delta": 0.012876830995082855, + "fcm_dpo/margin": 281.88671875, + "fcm_dpo/q_t": 0.4120361804962158, + "grad_norm": 28.710693359375, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.6545387506484985, + "logits/rejected": -0.635535478591919, + "logps/chosen": -451.828369140625, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -758.2076416015625, + "loss": 1.0973, + "margin_dpo/margin_mean": 281.8866882324219, + "margin_dpo/margin_std": 380.42938232421875, + "step": 453 + }, + { + "KL/chosen_KL_mean": -493.164306640625, + "KL/mean": -647.89892578125, + "KL/rejected_KL_mean": -802.6336669921875, + "KL/std": 370.6231689453125, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0013456812594085932, + "fcm_dpo/delta": -0.030497819185256958, + "fcm_dpo/margin": 309.46929931640625, + "fcm_dpo/q_t": 0.4019313454627991, + "grad_norm": 25.663516998291016, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.6898310780525208, + "logits/rejected": -0.676418662071228, + "logps/chosen": -554.1099853515625, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -887.5844116210938, + "loss": 1.0874, + "margin_dpo/margin_mean": 309.46929931640625, + "margin_dpo/margin_std": 427.5249938964844, + "step": 454 + }, + { + "KL/chosen_KL_mean": -418.80328369140625, + "KL/mean": -605.3193359375, + "KL/rejected_KL_mean": -791.8353881835938, + "KL/std": 368.81707763671875, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0012299984227865934, + "fcm_dpo/delta": -0.10926243662834167, + "fcm_dpo/margin": 373.0321044921875, + "fcm_dpo/q_t": 0.38569873571395874, + "grad_norm": 30.85955810546875, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.6721550226211548, + "logits/rejected": -0.6930463314056396, + "logps/chosen": -463.69000244140625, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -907.1368408203125, + "loss": 1.0505, + "margin_dpo/margin_mean": 373.0321044921875, + "margin_dpo/margin_std": 423.5682678222656, + "step": 455 + }, + { + "KL/chosen_KL_mean": -449.27508544921875, + "KL/mean": -635.3360595703125, + "KL/rejected_KL_mean": -821.3970947265625, + "KL/std": 394.14239501953125, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0010888128308579326, + "fcm_dpo/delta": -0.00909638125449419, + "fcm_dpo/margin": 372.12200927734375, + "fcm_dpo/q_t": 0.40475431084632874, + "grad_norm": 28.842674255371094, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.6700366139411926, + "logits/rejected": -0.6916704773902893, + "logps/chosen": -506.3118591308594, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -926.6149291992188, + "loss": 1.063, + "margin_dpo/margin_mean": 372.12200927734375, + "margin_dpo/margin_std": 389.7081298828125, + "step": 456 + }, + { + "KL/chosen_KL_mean": -442.96075439453125, + "KL/mean": -624.3992919921875, + "KL/rejected_KL_mean": -805.837646484375, + "KL/std": 381.4212646484375, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0010792326647788286, + "fcm_dpo/delta": 0.01370345987379551, + "fcm_dpo/margin": 362.876953125, + "fcm_dpo/q_t": 0.40837815403938293, + "grad_norm": 28.204694747924805, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.6881248950958252, + "logits/rejected": -0.68438720703125, + "logps/chosen": -497.20330810546875, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -890.947265625, + "loss": 1.0737, + "margin_dpo/margin_mean": 362.876953125, + "margin_dpo/margin_std": 379.9857177734375, + "step": 457 + }, + { + "KL/chosen_KL_mean": -499.1065368652344, + "KL/mean": -653.4949951171875, + "KL/rejected_KL_mean": -807.8834228515625, + "KL/std": 364.198974609375, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.0011557680554687977, + "fcm_dpo/delta": 0.06521759927272797, + "fcm_dpo/margin": 308.7769775390625, + "fcm_dpo/q_t": 0.4231434762477875, + "grad_norm": 32.81315231323242, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.7310912609100342, + "logits/rejected": -0.7260030508041382, + "logps/chosen": -554.515380859375, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -905.5667114257812, + "loss": 1.123, + "margin_dpo/margin_mean": 308.7769775390625, + "margin_dpo/margin_std": 427.2161560058594, + "step": 458 + }, + { + "KL/chosen_KL_mean": -529.4247436523438, + "KL/mean": -706.0894165039062, + "KL/rejected_KL_mean": -882.7540283203125, + "KL/std": 432.647216796875, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0011564318556338549, + "fcm_dpo/delta": -0.021825630217790604, + "fcm_dpo/margin": 353.329345703125, + "fcm_dpo/q_t": 0.40472641587257385, + "grad_norm": 32.47217559814453, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.7430839538574219, + "logits/rejected": -0.7598717212677002, + "logps/chosen": -575.9822387695312, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -968.922607421875, + "loss": 1.0937, + "margin_dpo/margin_mean": 353.329345703125, + "margin_dpo/margin_std": 485.6087341308594, + "step": 459 + }, + { + "KL/chosen_KL_mean": -571.9495239257812, + "KL/mean": -779.2578125, + "KL/rejected_KL_mean": -986.5660400390625, + "KL/std": 431.83953857421875, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0010669762268662453, + "fcm_dpo/delta": -0.07710142433643341, + "fcm_dpo/margin": 414.6165466308594, + "fcm_dpo/q_t": 0.3918076157569885, + "grad_norm": 30.98316764831543, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.830254316329956, + "logits/rejected": -0.8624805212020874, + "logps/chosen": -623.58447265625, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -1090.6854248046875, + "loss": 1.0514, + "margin_dpo/margin_mean": 414.6165466308594, + "margin_dpo/margin_std": 476.18707275390625, + "step": 460 + }, + { + "KL/chosen_KL_mean": -605.7537841796875, + "KL/mean": -766.1744384765625, + "KL/rejected_KL_mean": -926.59521484375, + "KL/std": 430.42913818359375, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0011395696783438325, + "fcm_dpo/delta": 0.04857172816991806, + "fcm_dpo/margin": 320.8414611816406, + "fcm_dpo/q_t": 0.41997230052948, + "grad_norm": 24.931995391845703, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.8122773170471191, + "logits/rejected": -0.8271867036819458, + "logps/chosen": -660.9356689453125, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1013.0721435546875, + "loss": 1.1224, + "margin_dpo/margin_mean": 320.8414306640625, + "margin_dpo/margin_std": 479.56878662109375, + "step": 461 + }, + { + "KL/chosen_KL_mean": -618.9212036132812, + "KL/mean": -744.9371337890625, + "KL/rejected_KL_mean": -870.9530639648438, + "KL/std": 416.94464111328125, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.001313498243689537, + "fcm_dpo/delta": 0.10240471363067627, + "fcm_dpo/margin": 252.0318603515625, + "fcm_dpo/q_t": 0.43281617760658264, + "grad_norm": 40.46854019165039, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.8632235527038574, + "logits/rejected": -0.8471617698669434, + "logps/chosen": -688.8492431640625, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -949.794189453125, + "loss": 1.1744, + "margin_dpo/margin_mean": 252.03182983398438, + "margin_dpo/margin_std": 479.5788269042969, + "step": 462 + }, + { + "KL/chosen_KL_mean": -603.7901611328125, + "KL/mean": -779.239990234375, + "KL/rejected_KL_mean": -954.6898803710938, + "KL/std": 431.516357421875, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.0012737201759591699, + "fcm_dpo/delta": -0.08446250110864639, + "fcm_dpo/margin": 350.899658203125, + "fcm_dpo/q_t": 0.39394649863243103, + "grad_norm": 37.86167526245117, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.8599103689193726, + "logits/rejected": -0.8692607879638672, + "logps/chosen": -659.0645751953125, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -1043.71484375, + "loss": 1.0823, + "margin_dpo/margin_mean": 350.899658203125, + "margin_dpo/margin_std": 486.9836120605469, + "step": 463 + }, + { + "KL/chosen_KL_mean": -577.685302734375, + "KL/mean": -836.02734375, + "KL/rejected_KL_mean": -1094.369384765625, + "KL/std": 506.09954833984375, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0008683481719344854, + "fcm_dpo/delta": -0.13791523873806, + "fcm_dpo/margin": 516.6839599609375, + "fcm_dpo/q_t": 0.37923869490623474, + "grad_norm": 35.81996536254883, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.8401867151260376, + "logits/rejected": -0.9024742841720581, + "logps/chosen": -628.59765625, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1196.858642578125, + "loss": 1.0364, + "margin_dpo/margin_mean": 516.6839599609375, + "margin_dpo/margin_std": 519.8881225585938, + "step": 464 + }, + { + "KL/chosen_KL_mean": -585.9309692382812, + "KL/mean": -794.7791748046875, + "KL/rejected_KL_mean": -1003.6273193359375, + "KL/std": 513.0881958007812, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0009388852631673217, + "fcm_dpo/delta": 0.007542621344327927, + "fcm_dpo/margin": 417.69635009765625, + "fcm_dpo/q_t": 0.4073883295059204, + "grad_norm": 24.084501266479492, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.7998400926589966, + "logits/rejected": -0.8214449882507324, + "logps/chosen": -646.0478515625, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -1117.5733642578125, + "loss": 1.0968, + "margin_dpo/margin_mean": 417.69635009765625, + "margin_dpo/margin_std": 542.79931640625, + "step": 465 + }, + { + "KL/chosen_KL_mean": -639.7386474609375, + "KL/mean": -817.8701171875, + "KL/rejected_KL_mean": -996.0015258789062, + "KL/std": 474.357666015625, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0009801898850128055, + "fcm_dpo/delta": 0.0780898779630661, + "fcm_dpo/margin": 356.26287841796875, + "fcm_dpo/q_t": 0.42381781339645386, + "grad_norm": 24.35711669921875, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.8397573232650757, + "logits/rejected": -0.840815007686615, + "logps/chosen": -692.6595458984375, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -1086.3170166015625, + "loss": 1.1348, + "margin_dpo/margin_mean": 356.26287841796875, + "margin_dpo/margin_std": 522.5521240234375, + "step": 466 + }, + { + "KL/chosen_KL_mean": -801.0821533203125, + "KL/mean": -986.1063232421875, + "KL/rejected_KL_mean": -1171.1304931640625, + "KL/std": 607.9132080078125, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0009247926063835621, + "fcm_dpo/delta": -0.06586673855781555, + "fcm_dpo/margin": 370.04833984375, + "fcm_dpo/q_t": 0.41659867763519287, + "grad_norm": 38.04802703857422, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.9427855610847473, + "logits/rejected": -0.934941291809082, + "logps/chosen": -879.7979736328125, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1273.99072265625, + "loss": 1.179, + "margin_dpo/margin_mean": 370.04833984375, + "margin_dpo/margin_std": 719.1068115234375, + "step": 467 + }, + { + "KL/chosen_KL_mean": -630.8428955078125, + "KL/mean": -867.4342041015625, + "KL/rejected_KL_mean": -1104.025634765625, + "KL/std": 526.3124389648438, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0008766874088905752, + "fcm_dpo/delta": -0.026581648737192154, + "fcm_dpo/margin": 473.18267822265625, + "fcm_dpo/q_t": 0.4034311771392822, + "grad_norm": 31.51830291748047, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.8827992677688599, + "logits/rejected": -0.8895440101623535, + "logps/chosen": -700.7782592773438, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1205.054443359375, + "loss": 1.0778, + "margin_dpo/margin_mean": 473.18267822265625, + "margin_dpo/margin_std": 609.1591796875, + "step": 468 + }, + { + "KL/chosen_KL_mean": -650.935302734375, + "KL/mean": -885.42578125, + "KL/rejected_KL_mean": -1119.916259765625, + "KL/std": 477.8541259765625, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0008414213662035763, + "fcm_dpo/delta": 0.0032104477286338806, + "fcm_dpo/margin": 468.98089599609375, + "fcm_dpo/q_t": 0.4071598947048187, + "grad_norm": 36.97335433959961, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.899463415145874, + "logits/rejected": -0.9126312732696533, + "logps/chosen": -719.0599975585938, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1224.70263671875, + "loss": 1.0813, + "margin_dpo/margin_mean": 468.98089599609375, + "margin_dpo/margin_std": 533.3753662109375, + "step": 469 + }, + { + "KL/chosen_KL_mean": -553.0526123046875, + "KL/mean": -755.4017333984375, + "KL/rejected_KL_mean": -957.7508544921875, + "KL/std": 496.32080078125, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0009463757742196321, + "fcm_dpo/delta": 0.022916335612535477, + "fcm_dpo/margin": 404.69830322265625, + "fcm_dpo/q_t": 0.41216540336608887, + "grad_norm": 23.679851531982422, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.8496056199073792, + "logits/rejected": -0.8591802716255188, + "logps/chosen": -596.844482421875, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1040.4537353515625, + "loss": 1.1079, + "margin_dpo/margin_mean": 404.69830322265625, + "margin_dpo/margin_std": 562.834228515625, + "step": 470 + }, + { + "KL/chosen_KL_mean": -711.413330078125, + "KL/mean": -888.9149169921875, + "KL/rejected_KL_mean": -1066.4163818359375, + "KL/std": 547.1355590820312, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.0008889293530955911, + "fcm_dpo/delta": -0.01932104118168354, + "fcm_dpo/margin": 355.0030517578125, + "fcm_dpo/q_t": 0.4268658757209778, + "grad_norm": 23.575862884521484, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.9526507258415222, + "logits/rejected": -0.9646092057228088, + "logps/chosen": -774.7528686523438, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1150.02685546875, + "loss": 1.1604, + "margin_dpo/margin_mean": 355.0030517578125, + "margin_dpo/margin_std": 585.3763427734375, + "step": 471 + }, + { + "KL/chosen_KL_mean": -698.0546875, + "KL/mean": -902.4146728515625, + "KL/rejected_KL_mean": -1106.774658203125, + "KL/std": 597.7765502929688, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0009217117330990732, + "fcm_dpo/delta": 0.0372597835958004, + "fcm_dpo/margin": 408.719970703125, + "fcm_dpo/q_t": 0.41907206177711487, + "grad_norm": 28.87525749206543, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.9155426025390625, + "logits/rejected": -0.8952895402908325, + "logps/chosen": -781.7208251953125, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1223.98388671875, + "loss": 1.1323, + "margin_dpo/margin_mean": 408.719970703125, + "margin_dpo/margin_std": 667.8204345703125, + "step": 472 + }, + { + "KL/chosen_KL_mean": -824.3423461914062, + "KL/mean": -920.7506103515625, + "KL/rejected_KL_mean": -1017.1588134765625, + "KL/std": 608.287353515625, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0009577026357874274, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 192.81646728515625, + "fcm_dpo/q_t": 0.4590563476085663, + "grad_norm": 100.81672668457031, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.9225292801856995, + "logits/rejected": -0.8966630697250366, + "logps/chosen": -887.83935546875, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1098.305419921875, + "loss": 1.3432, + "margin_dpo/margin_mean": 192.81646728515625, + "margin_dpo/margin_std": 805.4339599609375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -674.1361083984375, + "KL/mean": -858.5537109375, + "KL/rejected_KL_mean": -1042.971435546875, + "KL/std": 571.548583984375, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.0008901376277208328, + "fcm_dpo/delta": -0.0760503038764, + "fcm_dpo/margin": 368.83526611328125, + "fcm_dpo/q_t": 0.42114460468292236, + "grad_norm": 29.184354782104492, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.8849027156829834, + "logits/rejected": -0.8879544734954834, + "logps/chosen": -726.748046875, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1133.0517578125, + "loss": 1.1794, + "margin_dpo/margin_mean": 368.83526611328125, + "margin_dpo/margin_std": 658.4710693359375, + "step": 474 + }, + { + "KL/chosen_KL_mean": -489.204345703125, + "KL/mean": -716.9005737304688, + "KL/rejected_KL_mean": -944.5968627929688, + "KL/std": 447.86822509765625, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0008446001447737217, + "fcm_dpo/delta": 0.025007635354995728, + "fcm_dpo/margin": 455.3924560546875, + "fcm_dpo/q_t": 0.4114686846733093, + "grad_norm": 35.35837173461914, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.9520416855812073, + "logits/rejected": -0.979211688041687, + "logps/chosen": -531.6995849609375, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1034.6597900390625, + "loss": 1.0813, + "margin_dpo/margin_mean": 455.3924560546875, + "margin_dpo/margin_std": 503.96844482421875, + "step": 475 + }, + { + "KL/chosen_KL_mean": -619.6912231445312, + "KL/mean": -803.4817504882812, + "KL/rejected_KL_mean": -987.272216796875, + "KL/std": 514.7440185546875, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.0009646163089200854, + "fcm_dpo/delta": 0.06908048689365387, + "fcm_dpo/margin": 367.5810546875, + "fcm_dpo/q_t": 0.42334458231925964, + "grad_norm": 42.48811721801758, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.8881396055221558, + "logits/rejected": -0.904177188873291, + "logps/chosen": -662.640625, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1060.982421875, + "loss": 1.1232, + "margin_dpo/margin_mean": 367.5810546875, + "margin_dpo/margin_std": 530.9556884765625, + "step": 476 + }, + { + "KL/chosen_KL_mean": -674.9053955078125, + "KL/mean": -861.569580078125, + "KL/rejected_KL_mean": -1048.2337646484375, + "KL/std": 548.8595581054688, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0010453263530507684, + "fcm_dpo/delta": 0.012534024193882942, + "fcm_dpo/margin": 373.328369140625, + "fcm_dpo/q_t": 0.4125494360923767, + "grad_norm": 29.967121124267578, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.974365234375, + "logits/rejected": -0.9425362348556519, + "logps/chosen": -745.677978515625, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1124.37109375, + "loss": 1.145, + "margin_dpo/margin_mean": 373.3283386230469, + "margin_dpo/margin_std": 659.33203125, + "step": 477 + }, + { + "KL/chosen_KL_mean": -521.8460693359375, + "KL/mean": -730.65234375, + "KL/rejected_KL_mean": -939.4588012695312, + "KL/std": 461.107421875, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.001001200987957418, + "fcm_dpo/delta": -0.0321248322725296, + "fcm_dpo/margin": 417.6127014160156, + "fcm_dpo/q_t": 0.4019705057144165, + "grad_norm": 34.509376525878906, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.8223329782485962, + "logits/rejected": -0.8430248498916626, + "logps/chosen": -563.2865600585938, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1024.82080078125, + "loss": 1.081, + "margin_dpo/margin_mean": 417.61273193359375, + "margin_dpo/margin_std": 542.504150390625, + "step": 478 + }, + { + "KL/chosen_KL_mean": -683.9410400390625, + "KL/mean": -882.384033203125, + "KL/rejected_KL_mean": -1080.8271484375, + "KL/std": 582.9780883789062, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0010136861819773912, + "fcm_dpo/delta": -0.020786762237548828, + "fcm_dpo/margin": 396.88604736328125, + "fcm_dpo/q_t": 0.4105514883995056, + "grad_norm": 26.009479522705078, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.8741980791091919, + "logits/rejected": -0.9063311815261841, + "logps/chosen": -737.8489990234375, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1175.943359375, + "loss": 1.14, + "margin_dpo/margin_mean": 396.88604736328125, + "margin_dpo/margin_std": 691.9199829101562, + "step": 479 + }, + { + "KL/chosen_KL_mean": -823.0316772460938, + "KL/mean": -973.1109619140625, + "KL/rejected_KL_mean": -1123.190185546875, + "KL/std": 528.6361083984375, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0009059295989573002, + "fcm_dpo/delta": -0.010956360027194023, + "fcm_dpo/margin": 300.15850830078125, + "fcm_dpo/q_t": 0.43630528450012207, + "grad_norm": 46.897727966308594, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.9384490251541138, + "logits/rejected": -0.9253517389297485, + "logps/chosen": -881.71435546875, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1206.1226806640625, + "loss": 1.2068, + "margin_dpo/margin_mean": 300.15850830078125, + "margin_dpo/margin_std": 578.85498046875, + "step": 480 + }, + { + "KL/chosen_KL_mean": -666.0030517578125, + "KL/mean": -902.3043212890625, + "KL/rejected_KL_mean": -1138.605712890625, + "KL/std": 549.5169677734375, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0008731147972866893, + "fcm_dpo/delta": -0.021470826119184494, + "fcm_dpo/margin": 472.6026611328125, + "fcm_dpo/q_t": 0.4040641784667969, + "grad_norm": 27.580970764160156, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.9624149799346924, + "logits/rejected": -0.9881491661071777, + "logps/chosen": -720.96728515625, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1231.026123046875, + "loss": 1.0827, + "margin_dpo/margin_mean": 472.6026611328125, + "margin_dpo/margin_std": 616.1715087890625, + "step": 481 + }, + { + "KL/chosen_KL_mean": -744.04248046875, + "KL/mean": -855.4127197265625, + "KL/rejected_KL_mean": -966.782958984375, + "KL/std": 563.21630859375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.0009502337779849768, + "fcm_dpo/delta": 0.09700674563646317, + "fcm_dpo/margin": 222.74044799804688, + "fcm_dpo/q_t": 0.4541202783584595, + "grad_norm": 35.13661575317383, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.8878397941589355, + "logits/rejected": -0.8769594430923462, + "logps/chosen": -811.595947265625, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1054.37255859375, + "loss": 1.2788, + "margin_dpo/margin_mean": 222.740478515625, + "margin_dpo/margin_std": 668.005859375, + "step": 482 + }, + { + "KL/chosen_KL_mean": -656.9723510742188, + "KL/mean": -896.6019897460938, + "KL/rejected_KL_mean": -1136.2315673828125, + "KL/std": 552.5379028320312, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0009095891728065908, + "fcm_dpo/delta": -0.07079771906137466, + "fcm_dpo/margin": 479.25921630859375, + "fcm_dpo/q_t": 0.3960753083229065, + "grad_norm": 31.120384216308594, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.8999602198600769, + "logits/rejected": -0.9098832607269287, + "logps/chosen": -720.3021850585938, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1232.0185546875, + "loss": 1.0695, + "margin_dpo/margin_mean": 479.2591857910156, + "margin_dpo/margin_std": 622.5853271484375, + "step": 483 + }, + { + "KL/chosen_KL_mean": -550.8277587890625, + "KL/mean": -768.7025146484375, + "KL/rejected_KL_mean": -986.5772705078125, + "KL/std": 529.3361206054688, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0009340384276583791, + "fcm_dpo/delta": -0.01913183555006981, + "fcm_dpo/margin": 435.74957275390625, + "fcm_dpo/q_t": 0.40439409017562866, + "grad_norm": 38.8234748840332, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.9227362871170044, + "logits/rejected": -0.9442682266235352, + "logps/chosen": -609.9659423828125, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1070.94873046875, + "loss": 1.0868, + "margin_dpo/margin_mean": 435.74951171875, + "margin_dpo/margin_std": 546.264892578125, + "step": 484 + }, + { + "KL/chosen_KL_mean": -558.8453369140625, + "KL/mean": -764.8858642578125, + "KL/rejected_KL_mean": -970.9264526367188, + "KL/std": 525.61865234375, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0009324835264123976, + "fcm_dpo/delta": 0.020539814606308937, + "fcm_dpo/margin": 412.0811767578125, + "fcm_dpo/q_t": 0.41461336612701416, + "grad_norm": 38.73042678833008, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.8426438570022583, + "logits/rejected": -0.8627835512161255, + "logps/chosen": -617.6948852539062, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1074.29052734375, + "loss": 1.1164, + "margin_dpo/margin_mean": 412.0811462402344, + "margin_dpo/margin_std": 618.0966186523438, + "step": 485 + }, + { + "KL/chosen_KL_mean": -644.1904296875, + "KL/mean": -887.1826782226562, + "KL/rejected_KL_mean": -1130.1748046875, + "KL/std": 590.3195190429688, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0008638256113044918, + "fcm_dpo/delta": -0.034225355833768845, + "fcm_dpo/margin": 485.98443603515625, + "fcm_dpo/q_t": 0.4027414321899414, + "grad_norm": 39.733394622802734, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.9089898467063904, + "logits/rejected": -0.9190336465835571, + "logps/chosen": -699.4500732421875, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1222.314208984375, + "loss": 1.0991, + "margin_dpo/margin_mean": 485.9844665527344, + "margin_dpo/margin_std": 727.2791748046875, + "step": 486 + }, + { + "KL/chosen_KL_mean": -683.022216796875, + "KL/mean": -864.6463012695312, + "KL/rejected_KL_mean": -1046.2703857421875, + "KL/std": 544.7926025390625, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0009704558178782463, + "fcm_dpo/delta": 0.06513335555791855, + "fcm_dpo/margin": 363.2481689453125, + "fcm_dpo/q_t": 0.4213024377822876, + "grad_norm": 40.45846939086914, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.9383945465087891, + "logits/rejected": -0.9428784847259521, + "logps/chosen": -736.0855712890625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1138.689208984375, + "loss": 1.1385, + "margin_dpo/margin_mean": 363.2481689453125, + "margin_dpo/margin_std": 577.7797241210938, + "step": 487 + }, + { + "KL/chosen_KL_mean": -545.5859985351562, + "KL/mean": -734.630126953125, + "KL/rejected_KL_mean": -923.67431640625, + "KL/std": 473.2789306640625, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0010280082933604717, + "fcm_dpo/delta": 0.012521795928478241, + "fcm_dpo/margin": 378.0882873535156, + "fcm_dpo/q_t": 0.41051536798477173, + "grad_norm": 22.53078269958496, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.8381102681159973, + "logits/rejected": -0.8391183614730835, + "logps/chosen": -597.814208984375, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1007.680908203125, + "loss": 1.0927, + "margin_dpo/margin_mean": 378.0882873535156, + "margin_dpo/margin_std": 473.35400390625, + "step": 488 + }, + { + "KL/chosen_KL_mean": -499.30194091796875, + "KL/mean": -672.4368286132812, + "KL/rejected_KL_mean": -845.5716552734375, + "KL/std": 457.94464111328125, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.00108251697383821, + "fcm_dpo/delta": 0.03405838832259178, + "fcm_dpo/margin": 346.2696838378906, + "fcm_dpo/q_t": 0.41736501455307007, + "grad_norm": 24.369661331176758, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.8864847421646118, + "logits/rejected": -0.8804982900619507, + "logps/chosen": -555.2916259765625, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -924.9697265625, + "loss": 1.1145, + "margin_dpo/margin_mean": 346.2696838378906, + "margin_dpo/margin_std": 504.04510498046875, + "step": 489 + }, + { + "KL/chosen_KL_mean": -606.05126953125, + "KL/mean": -801.578125, + "KL/rejected_KL_mean": -997.1050415039062, + "KL/std": 577.8994140625, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0010517244227230549, + "fcm_dpo/delta": -0.019802920520305634, + "fcm_dpo/margin": 391.0537414550781, + "fcm_dpo/q_t": 0.4080085754394531, + "grad_norm": 39.334407806396484, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.8914596438407898, + "logits/rejected": -0.9301384687423706, + "logps/chosen": -658.4176025390625, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1107.51416015625, + "loss": 1.1285, + "margin_dpo/margin_mean": 391.05377197265625, + "margin_dpo/margin_std": 660.88671875, + "step": 490 + }, + { + "KL/chosen_KL_mean": -556.3938598632812, + "KL/mean": -685.6237182617188, + "KL/rejected_KL_mean": -814.8536376953125, + "KL/std": 486.525634765625, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0012323971604928374, + "fcm_dpo/delta": 0.1136833056807518, + "fcm_dpo/margin": 258.459716796875, + "fcm_dpo/q_t": 0.4345433712005615, + "grad_norm": 34.78307342529297, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.8827264904975891, + "logits/rejected": -0.8719267845153809, + "logps/chosen": -616.5101318359375, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -888.1264038085938, + "loss": 1.175, + "margin_dpo/margin_mean": 258.4597473144531, + "margin_dpo/margin_std": 484.7692565917969, + "step": 491 + }, + { + "KL/chosen_KL_mean": -558.9266357421875, + "KL/mean": -683.21923828125, + "KL/rejected_KL_mean": -807.511962890625, + "KL/std": 487.9819030761719, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0014084703288972378, + "fcm_dpo/delta": 0.07727247476577759, + "fcm_dpo/margin": 248.58529663085938, + "fcm_dpo/q_t": 0.43000033497810364, + "grad_norm": 36.50166320800781, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.8998019695281982, + "logits/rejected": -0.8745296001434326, + "logps/chosen": -612.9207763671875, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -880.1715087890625, + "loss": 1.209, + "margin_dpo/margin_mean": 248.58529663085938, + "margin_dpo/margin_std": 559.0795288085938, + "step": 492 + }, + { + "KL/chosen_KL_mean": -606.264892578125, + "KL/mean": -745.5481567382812, + "KL/rejected_KL_mean": -884.8314208984375, + "KL/std": 519.5693359375, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0013839509338140488, + "fcm_dpo/delta": 0.0048267170786857605, + "fcm_dpo/margin": 278.56646728515625, + "fcm_dpo/q_t": 0.412563681602478, + "grad_norm": 41.9803466796875, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -0.9656150341033936, + "logits/rejected": -0.9288890361785889, + "logps/chosen": -681.7621459960938, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -972.1544189453125, + "loss": 1.1628, + "margin_dpo/margin_mean": 278.56646728515625, + "margin_dpo/margin_std": 508.8519287109375, + "step": 493 + }, + { + "KL/chosen_KL_mean": -456.39111328125, + "KL/mean": -672.3492431640625, + "KL/rejected_KL_mean": -888.307373046875, + "KL/std": 460.58941650390625, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.001108669675886631, + "fcm_dpo/delta": -0.17697980999946594, + "fcm_dpo/margin": 431.91619873046875, + "fcm_dpo/q_t": 0.37332338094711304, + "grad_norm": 37.33299255371094, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.8224232196807861, + "logits/rejected": -0.8543698191642761, + "logps/chosen": -497.7503662109375, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -974.398681640625, + "loss": 1.0276, + "margin_dpo/margin_mean": 431.91619873046875, + "margin_dpo/margin_std": 471.99713134765625, + "step": 494 + }, + { + "KL/chosen_KL_mean": -516.5604248046875, + "KL/mean": -703.7841796875, + "KL/rejected_KL_mean": -891.0079956054688, + "KL/std": 482.6785888671875, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0010678664548322558, + "fcm_dpo/delta": -4.4368207454681396e-05, + "fcm_dpo/margin": 374.44757080078125, + "fcm_dpo/q_t": 0.4114726483821869, + "grad_norm": 35.54365539550781, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -0.9496263265609741, + "logits/rejected": -0.9522305727005005, + "logps/chosen": -580.095458984375, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -982.432373046875, + "loss": 1.1106, + "margin_dpo/margin_mean": 374.44757080078125, + "margin_dpo/margin_std": 582.944580078125, + "step": 495 + }, + { + "KL/chosen_KL_mean": -618.7561645507812, + "KL/mean": -733.871337890625, + "KL/rejected_KL_mean": -848.986572265625, + "KL/std": 380.02642822265625, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0013148835860192776, + "fcm_dpo/delta": 0.13359305262565613, + "fcm_dpo/margin": 230.23040771484375, + "fcm_dpo/q_t": 0.43739110231399536, + "grad_norm": 59.39929962158203, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.9213845729827881, + "logits/rejected": -0.8853435516357422, + "logps/chosen": -691.3480834960938, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -933.31591796875, + "loss": 1.1831, + "margin_dpo/margin_mean": 230.23040771484375, + "margin_dpo/margin_std": 436.0897216796875, + "step": 496 + }, + { + "KL/chosen_KL_mean": -592.6030883789062, + "KL/mean": -699.4375, + "KL/rejected_KL_mean": -806.27197265625, + "KL/std": 449.3829345703125, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0013049728004261851, + "fcm_dpo/delta": -0.028186559677124023, + "fcm_dpo/margin": 213.6688232421875, + "fcm_dpo/q_t": 0.4349159598350525, + "grad_norm": 47.662078857421875, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.8769658803939819, + "logits/rejected": -0.8549270629882812, + "logps/chosen": -651.1970825195312, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -882.560302734375, + "loss": 1.2114, + "margin_dpo/margin_mean": 213.6688232421875, + "margin_dpo/margin_std": 445.9394836425781, + "step": 497 + }, + { + "KL/chosen_KL_mean": -555.0482788085938, + "KL/mean": -724.8245849609375, + "KL/rejected_KL_mean": -894.6010131835938, + "KL/std": 492.70159912109375, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0012368912575766444, + "fcm_dpo/delta": -0.03477949649095535, + "fcm_dpo/margin": 339.552734375, + "fcm_dpo/q_t": 0.4025610685348511, + "grad_norm": 32.19281768798828, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.93885338306427, + "logits/rejected": -0.9247368574142456, + "logps/chosen": -626.25390625, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -978.55908203125, + "loss": 1.1152, + "margin_dpo/margin_mean": 339.552734375, + "margin_dpo/margin_std": 550.4183349609375, + "step": 498 + }, + { + "KL/chosen_KL_mean": -468.7248840332031, + "KL/mean": -666.8063354492188, + "KL/rejected_KL_mean": -864.8878173828125, + "KL/std": 467.00946044921875, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0010947990231215954, + "fcm_dpo/delta": -0.062151163816452026, + "fcm_dpo/margin": 396.1629638671875, + "fcm_dpo/q_t": 0.39568614959716797, + "grad_norm": 36.04325485229492, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.8788598775863647, + "logits/rejected": -0.9086207151412964, + "logps/chosen": -519.9801025390625, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -965.966552734375, + "loss": 1.0746, + "margin_dpo/margin_mean": 396.1629638671875, + "margin_dpo/margin_std": 527.5292358398438, + "step": 499 + }, + { + "KL/chosen_KL_mean": -586.3583374023438, + "KL/mean": -761.30419921875, + "KL/rejected_KL_mean": -936.25, + "KL/std": 430.88531494140625, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0010993874166160822, + "fcm_dpo/delta": 0.024825483560562134, + "fcm_dpo/margin": 349.8916931152344, + "fcm_dpo/q_t": 0.4142201840877533, + "grad_norm": 32.015804290771484, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.9118782877922058, + "logits/rejected": -0.9368890523910522, + "logps/chosen": -643.3857421875, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1030.1842041015625, + "loss": 1.1177, + "margin_dpo/margin_mean": 349.8916931152344, + "margin_dpo/margin_std": 529.33837890625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -496.50091552734375, + "KL/mean": -679.2710571289062, + "KL/rejected_KL_mean": -862.041259765625, + "KL/std": 453.33331298828125, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.001129016513004899, + "fcm_dpo/delta": -0.023394692689180374, + "fcm_dpo/margin": 365.5402526855469, + "fcm_dpo/q_t": 0.4061323404312134, + "grad_norm": 33.857051849365234, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.8639429211616516, + "logits/rejected": -0.8466420769691467, + "logps/chosen": -550.8604736328125, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -942.1979370117188, + "loss": 1.1061, + "margin_dpo/margin_mean": 365.540283203125, + "margin_dpo/margin_std": 553.092041015625, + "step": 501 + }, + { + "KL/chosen_KL_mean": -463.086181640625, + "KL/mean": -645.1473388671875, + "KL/rejected_KL_mean": -827.20849609375, + "KL/std": 410.1119689941406, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0010994459735229611, + "fcm_dpo/delta": -0.0017848927527666092, + "fcm_dpo/margin": 364.122314453125, + "fcm_dpo/q_t": 0.40732836723327637, + "grad_norm": 22.971820831298828, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.8409848809242249, + "logits/rejected": -0.8154308795928955, + "logps/chosen": -530.6866455078125, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -910.1572265625, + "loss": 1.0803, + "margin_dpo/margin_mean": 364.1223449707031, + "margin_dpo/margin_std": 437.77301025390625, + "step": 502 + }, + { + "KL/chosen_KL_mean": -455.322998046875, + "KL/mean": -625.4855346679688, + "KL/rejected_KL_mean": -795.6480712890625, + "KL/std": 388.8305358886719, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.00111018109600991, + "fcm_dpo/delta": 0.03548618406057358, + "fcm_dpo/margin": 340.3250427246094, + "fcm_dpo/q_t": 0.4157389998435974, + "grad_norm": 23.465837478637695, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.8313039541244507, + "logits/rejected": -0.8144758939743042, + "logps/chosen": -510.40142822265625, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -878.1535034179688, + "loss": 1.0957, + "margin_dpo/margin_mean": 340.3250427246094, + "margin_dpo/margin_std": 424.44854736328125, + "step": 503 + }, + { + "KL/chosen_KL_mean": -486.9976806640625, + "KL/mean": -691.9973754882812, + "KL/rejected_KL_mean": -896.9971313476562, + "KL/std": 449.76153564453125, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.001057411776855588, + "fcm_dpo/delta": -0.059654317796230316, + "fcm_dpo/margin": 409.99945068359375, + "fcm_dpo/q_t": 0.39566490054130554, + "grad_norm": 23.978740692138672, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.8678739070892334, + "logits/rejected": -0.8862636089324951, + "logps/chosen": -546.96337890625, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1000.75927734375, + "loss": 1.0674, + "margin_dpo/margin_mean": 409.9994812011719, + "margin_dpo/margin_std": 505.0823974609375, + "step": 504 + }, + { + "KL/chosen_KL_mean": -583.2789306640625, + "KL/mean": -693.8592529296875, + "KL/rejected_KL_mean": -804.4395751953125, + "KL/std": 454.05352783203125, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0013018188765272498, + "fcm_dpo/delta": 0.1528121829032898, + "fcm_dpo/margin": 221.16064453125, + "fcm_dpo/q_t": 0.44206857681274414, + "grad_norm": 44.39717102050781, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.9133349657058716, + "logits/rejected": -0.8864049911499023, + "logps/chosen": -659.433837890625, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -893.02490234375, + "loss": 1.1987, + "margin_dpo/margin_mean": 221.16065979003906, + "margin_dpo/margin_std": 430.33538818359375, + "step": 505 + }, + { + "KL/chosen_KL_mean": -461.5542907714844, + "KL/mean": -646.5904541015625, + "KL/rejected_KL_mean": -831.626708984375, + "KL/std": 457.093505859375, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0011723710922524333, + "fcm_dpo/delta": -0.07656194269657135, + "fcm_dpo/margin": 370.07244873046875, + "fcm_dpo/q_t": 0.3941624164581299, + "grad_norm": 31.93734359741211, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.8899002075195312, + "logits/rejected": -0.8990935683250427, + "logps/chosen": -510.5147705078125, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -910.041748046875, + "loss": 1.083, + "margin_dpo/margin_mean": 370.07244873046875, + "margin_dpo/margin_std": 513.3609619140625, + "step": 506 + }, + { + "KL/chosen_KL_mean": -530.3336791992188, + "KL/mean": -686.3297729492188, + "KL/rejected_KL_mean": -842.3258666992188, + "KL/std": 511.7806701660156, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0012264869874343276, + "fcm_dpo/delta": 0.028089439496397972, + "fcm_dpo/margin": 311.9921569824219, + "fcm_dpo/q_t": 0.41973698139190674, + "grad_norm": 32.90348434448242, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.8918719291687012, + "logits/rejected": -0.889752984046936, + "logps/chosen": -597.1351928710938, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -937.69873046875, + "loss": 1.1451, + "margin_dpo/margin_mean": 311.9921569824219, + "margin_dpo/margin_std": 565.5931396484375, + "step": 507 + }, + { + "KL/chosen_KL_mean": -583.1611328125, + "KL/mean": -721.2760009765625, + "KL/rejected_KL_mean": -859.390869140625, + "KL/std": 509.71832275390625, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0013669917825609446, + "fcm_dpo/delta": 0.031157607212662697, + "fcm_dpo/margin": 276.22967529296875, + "fcm_dpo/q_t": 0.4213915169239044, + "grad_norm": 53.349178314208984, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.8866854906082153, + "logits/rejected": -0.883845329284668, + "logps/chosen": -654.464599609375, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -955.0184326171875, + "loss": 1.1942, + "margin_dpo/margin_mean": 276.2297058105469, + "margin_dpo/margin_std": 601.1476440429688, + "step": 508 + }, + { + "KL/chosen_KL_mean": -447.341552734375, + "KL/mean": -596.3013916015625, + "KL/rejected_KL_mean": -745.26123046875, + "KL/std": 366.6440734863281, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0013803851325064898, + "fcm_dpo/delta": -0.028025750070810318, + "fcm_dpo/margin": 297.9196472167969, + "fcm_dpo/q_t": 0.4056088328361511, + "grad_norm": 36.49995803833008, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.8121344447135925, + "logits/rejected": -0.8039395809173584, + "logps/chosen": -511.1605224609375, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -828.5177001953125, + "loss": 1.1197, + "margin_dpo/margin_mean": 297.9196472167969, + "margin_dpo/margin_std": 458.0248107910156, + "step": 509 + }, + { + "KL/chosen_KL_mean": -541.6021728515625, + "KL/mean": -733.678466796875, + "KL/rejected_KL_mean": -925.7547607421875, + "KL/std": 435.32470703125, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0011522183194756508, + "fcm_dpo/delta": -0.07610686868429184, + "fcm_dpo/margin": 384.152587890625, + "fcm_dpo/q_t": 0.3902145326137543, + "grad_norm": 38.18740463256836, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.7922756671905518, + "logits/rejected": -0.8119780421257019, + "logps/chosen": -593.4805908203125, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1028.5198974609375, + "loss": 1.0452, + "margin_dpo/margin_mean": 384.152587890625, + "margin_dpo/margin_std": 409.4080810546875, + "step": 510 + }, + { + "KL/chosen_KL_mean": -491.181884765625, + "KL/mean": -649.146484375, + "KL/rejected_KL_mean": -807.1109619140625, + "KL/std": 461.1341247558594, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0011797586921602488, + "fcm_dpo/delta": 0.0413035973906517, + "fcm_dpo/margin": 315.92913818359375, + "fcm_dpo/q_t": 0.41845548152923584, + "grad_norm": 29.213090896606445, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.8354673385620117, + "logits/rejected": -0.8356435298919678, + "logps/chosen": -551.4199829101562, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -899.9677734375, + "loss": 1.1461, + "margin_dpo/margin_mean": 315.92913818359375, + "margin_dpo/margin_std": 560.2075805664062, + "step": 511 + }, + { + "KL/chosen_KL_mean": -411.56103515625, + "KL/mean": -608.6541137695312, + "KL/rejected_KL_mean": -805.7471923828125, + "KL/std": 410.18994140625, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.0010748871136456728, + "fcm_dpo/delta": -0.04578953981399536, + "fcm_dpo/margin": 394.1861572265625, + "fcm_dpo/q_t": 0.3963681757450104, + "grad_norm": 52.93358612060547, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.8261533975601196, + "logits/rejected": -0.8413060307502747, + "logps/chosen": -466.466552734375, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -887.623046875, + "loss": 1.0543, + "margin_dpo/margin_mean": 394.1861572265625, + "margin_dpo/margin_std": 417.51800537109375, + "step": 512 + }, + { + "KL/chosen_KL_mean": -532.857421875, + "KL/mean": -685.5048828125, + "KL/rejected_KL_mean": -838.1524658203125, + "KL/std": 409.397705078125, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0009999822359532118, + "fcm_dpo/delta": -0.038525860756635666, + "fcm_dpo/margin": 305.2950134277344, + "fcm_dpo/q_t": 0.42581143975257874, + "grad_norm": 34.72964859008789, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.8888071775436401, + "logits/rejected": -0.8702949285507202, + "logps/chosen": -597.7738647460938, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -914.2149047851562, + "loss": 1.1629, + "margin_dpo/margin_mean": 305.2950134277344, + "margin_dpo/margin_std": 478.2789306640625, + "step": 513 + }, + { + "KL/chosen_KL_mean": -564.9537353515625, + "KL/mean": -707.5296630859375, + "KL/rejected_KL_mean": -850.1055908203125, + "KL/std": 448.4078063964844, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.0011754620354622602, + "fcm_dpo/delta": 0.09163933992385864, + "fcm_dpo/margin": 285.15191650390625, + "fcm_dpo/q_t": 0.4304603934288025, + "grad_norm": 32.26676559448242, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.8373386859893799, + "logits/rejected": -0.8227113485336304, + "logps/chosen": -639.1832885742188, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -929.0511474609375, + "loss": 1.1577, + "margin_dpo/margin_mean": 285.15191650390625, + "margin_dpo/margin_std": 502.1484375, + "step": 514 + }, + { + "KL/chosen_KL_mean": -494.32861328125, + "KL/mean": -618.3717041015625, + "KL/rejected_KL_mean": -742.414794921875, + "KL/std": 406.7989501953125, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0013804570771753788, + "fcm_dpo/delta": 0.08468376100063324, + "fcm_dpo/margin": 248.086181640625, + "fcm_dpo/q_t": 0.4263339042663574, + "grad_norm": 40.3935546875, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.8123036623001099, + "logits/rejected": -0.8226903676986694, + "logps/chosen": -544.730224609375, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -829.5125122070312, + "loss": 1.1724, + "margin_dpo/margin_mean": 248.086181640625, + "margin_dpo/margin_std": 483.9718322753906, + "step": 515 + }, + { + "KL/chosen_KL_mean": -526.0533447265625, + "KL/mean": -701.6831665039062, + "KL/rejected_KL_mean": -877.3131103515625, + "KL/std": 438.8985900878906, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.0012882768642157316, + "fcm_dpo/delta": -0.0951680988073349, + "fcm_dpo/margin": 351.25970458984375, + "fcm_dpo/q_t": 0.3881225287914276, + "grad_norm": 36.28194046020508, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.9183084964752197, + "logits/rejected": -0.9395666122436523, + "logps/chosen": -590.702880859375, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -989.0354614257812, + "loss": 1.0661, + "margin_dpo/margin_mean": 351.25970458984375, + "margin_dpo/margin_std": 446.1391906738281, + "step": 516 + }, + { + "KL/chosen_KL_mean": -552.42333984375, + "KL/mean": -758.86474609375, + "KL/rejected_KL_mean": -965.3060302734375, + "KL/std": 489.52093505859375, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0010481756180524826, + "fcm_dpo/delta": -0.0622154101729393, + "fcm_dpo/margin": 412.88262939453125, + "fcm_dpo/q_t": 0.39706072211265564, + "grad_norm": 25.22186279296875, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.8430509567260742, + "logits/rejected": -0.837881326675415, + "logps/chosen": -613.3369140625, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1054.38916015625, + "loss": 1.0697, + "margin_dpo/margin_mean": 412.88262939453125, + "margin_dpo/margin_std": 533.8587646484375, + "step": 517 + }, + { + "KL/chosen_KL_mean": -519.568603515625, + "KL/mean": -694.2239990234375, + "KL/rejected_KL_mean": -868.8794555664062, + "KL/std": 415.3473815917969, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.0010671718046069145, + "fcm_dpo/delta": 0.0419648140668869, + "fcm_dpo/margin": 349.3109130859375, + "fcm_dpo/q_t": 0.4160323143005371, + "grad_norm": 33.90839385986328, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.875517725944519, + "logits/rejected": -0.8914189338684082, + "logps/chosen": -577.0244140625, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -954.192138671875, + "loss": 1.112, + "margin_dpo/margin_mean": 349.3109130859375, + "margin_dpo/margin_std": 470.837158203125, + "step": 518 + }, + { + "KL/chosen_KL_mean": -517.7393798828125, + "KL/mean": -643.4759521484375, + "KL/rejected_KL_mean": -769.21240234375, + "KL/std": 388.196044921875, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0011561332503333688, + "fcm_dpo/delta": 0.02302495203912258, + "fcm_dpo/margin": 251.4730224609375, + "fcm_dpo/q_t": 0.4338497221469879, + "grad_norm": 42.36438751220703, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.889582097530365, + "logits/rejected": -0.8709315061569214, + "logps/chosen": -591.802734375, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -873.6566162109375, + "loss": 1.1946, + "margin_dpo/margin_mean": 251.4730224609375, + "margin_dpo/margin_std": 490.158447265625, + "step": 519 + }, + { + "KL/chosen_KL_mean": -572.123291015625, + "KL/mean": -733.942138671875, + "KL/rejected_KL_mean": -895.7611083984375, + "KL/std": 462.9925537109375, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.001188506605103612, + "fcm_dpo/delta": 0.023475412279367447, + "fcm_dpo/margin": 323.6377868652344, + "fcm_dpo/q_t": 0.41749107837677, + "grad_norm": 31.222883224487305, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.8482314348220825, + "logits/rejected": -0.8507933616638184, + "logps/chosen": -642.423095703125, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -995.742431640625, + "loss": 1.1305, + "margin_dpo/margin_mean": 323.6378173828125, + "margin_dpo/margin_std": 529.4666748046875, + "step": 520 + }, + { + "KL/chosen_KL_mean": -498.2612609863281, + "KL/mean": -677.0108032226562, + "KL/rejected_KL_mean": -855.76025390625, + "KL/std": 452.33221435546875, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.001218126737512648, + "fcm_dpo/delta": -0.07213665544986725, + "fcm_dpo/margin": 357.4990234375, + "fcm_dpo/q_t": 0.39487892389297485, + "grad_norm": 34.73341369628906, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.8869306445121765, + "logits/rejected": -0.8860381841659546, + "logps/chosen": -556.4041748046875, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -939.0408935546875, + "loss": 1.0806, + "margin_dpo/margin_mean": 357.4990234375, + "margin_dpo/margin_std": 450.37408447265625, + "step": 521 + }, + { + "KL/chosen_KL_mean": -573.8514404296875, + "KL/mean": -739.8829345703125, + "KL/rejected_KL_mean": -905.91455078125, + "KL/std": 490.18310546875, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0011361883953213692, + "fcm_dpo/delta": 0.036084454506635666, + "fcm_dpo/margin": 332.06304931640625, + "fcm_dpo/q_t": 0.41713184118270874, + "grad_norm": 28.38233757019043, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.9024146199226379, + "logits/rejected": -0.9035881757736206, + "logps/chosen": -620.3991088867188, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -971.928466796875, + "loss": 1.128, + "margin_dpo/margin_mean": 332.0630798339844, + "margin_dpo/margin_std": 528.5509033203125, + "step": 522 + }, + { + "KL/chosen_KL_mean": -621.0657348632812, + "KL/mean": -792.4852905273438, + "KL/rejected_KL_mean": -963.9049072265625, + "KL/std": 544.6927490234375, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0011524436995387077, + "fcm_dpo/delta": 0.007822713814675808, + "fcm_dpo/margin": 342.8391418457031, + "fcm_dpo/q_t": 0.4123551845550537, + "grad_norm": 35.35528564453125, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.9692133665084839, + "logits/rejected": -0.9667251110076904, + "logps/chosen": -682.8353271484375, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1047.666259765625, + "loss": 1.1316, + "margin_dpo/margin_mean": 342.8391418457031, + "margin_dpo/margin_std": 576.16796875, + "step": 523 + }, + { + "KL/chosen_KL_mean": -588.4208984375, + "KL/mean": -767.6956787109375, + "KL/rejected_KL_mean": -946.9705810546875, + "KL/std": 500.2874755859375, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0011021224781870842, + "fcm_dpo/delta": -0.0017808079719543457, + "fcm_dpo/margin": 358.5496826171875, + "fcm_dpo/q_t": 0.40673327445983887, + "grad_norm": 34.97015380859375, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.9925479888916016, + "logits/rejected": -0.9842585325241089, + "logps/chosen": -666.492919921875, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1028.2725830078125, + "loss": 1.1098, + "margin_dpo/margin_mean": 358.5496520996094, + "margin_dpo/margin_std": 513.2574462890625, + "step": 524 + }, + { + "KL/chosen_KL_mean": -575.912109375, + "KL/mean": -803.885986328125, + "KL/rejected_KL_mean": -1031.85986328125, + "KL/std": 526.4970092773438, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0009674688335508108, + "fcm_dpo/delta": -0.09075939655303955, + "fcm_dpo/margin": 455.9477233886719, + "fcm_dpo/q_t": 0.39121919870376587, + "grad_norm": 35.0758056640625, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.89338219165802, + "logits/rejected": -0.9303810596466064, + "logps/chosen": -626.739990234375, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1131.912841796875, + "loss": 1.06, + "margin_dpo/margin_mean": 455.94775390625, + "margin_dpo/margin_std": 559.2699584960938, + "step": 525 + }, + { + "KL/chosen_KL_mean": -609.7196044921875, + "KL/mean": -831.7645874023438, + "KL/rejected_KL_mean": -1053.8095703125, + "KL/std": 516.262939453125, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0009489007061347365, + "fcm_dpo/delta": -0.03767908737063408, + "fcm_dpo/margin": 444.0899658203125, + "fcm_dpo/q_t": 0.3990199565887451, + "grad_norm": 31.720500946044922, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.921222984790802, + "logits/rejected": -0.9137049317359924, + "logps/chosen": -672.8868408203125, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1140.118896484375, + "loss": 1.0826, + "margin_dpo/margin_mean": 444.0899658203125, + "margin_dpo/margin_std": 581.716064453125, + "step": 526 + }, + { + "KL/chosen_KL_mean": -647.0347290039062, + "KL/mean": -790.537841796875, + "KL/rejected_KL_mean": -934.0408935546875, + "KL/std": 527.8538208007812, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0010856003500521183, + "fcm_dpo/delta": 0.1282484382390976, + "fcm_dpo/margin": 287.0062255859375, + "fcm_dpo/q_t": 0.4351726174354553, + "grad_norm": 38.458126068115234, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.1125977039337158, + "logits/rejected": -1.0749619007110596, + "logps/chosen": -718.1634521484375, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1012.3834838867188, + "loss": 1.1598, + "margin_dpo/margin_mean": 287.0062255859375, + "margin_dpo/margin_std": 455.161376953125, + "step": 527 + }, + { + "KL/chosen_KL_mean": -650.195068359375, + "KL/mean": -832.568359375, + "KL/rejected_KL_mean": -1014.9417724609375, + "KL/std": 509.81414794921875, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0011271900730207562, + "fcm_dpo/delta": -0.019259048625826836, + "fcm_dpo/margin": 364.7466735839844, + "fcm_dpo/q_t": 0.4092450439929962, + "grad_norm": 36.67366409301758, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -0.981177568435669, + "logits/rejected": -0.9641728401184082, + "logps/chosen": -718.2845458984375, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1108.851806640625, + "loss": 1.1235, + "margin_dpo/margin_mean": 364.7467041015625, + "margin_dpo/margin_std": 606.7515258789062, + "step": 528 + }, + { + "KL/chosen_KL_mean": -796.822509765625, + "KL/mean": -921.7469482421875, + "KL/rejected_KL_mean": -1046.67138671875, + "KL/std": 494.61181640625, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0011494512436911464, + "fcm_dpo/delta": 0.01170584000647068, + "fcm_dpo/margin": 249.8488006591797, + "fcm_dpo/q_t": 0.4349808990955353, + "grad_norm": 53.19778823852539, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.048140525817871, + "logits/rejected": -1.0602138042449951, + "logps/chosen": -852.3974609375, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1135.88037109375, + "loss": 1.2429, + "margin_dpo/margin_mean": 249.84881591796875, + "margin_dpo/margin_std": 625.3258056640625, + "step": 529 + }, + { + "KL/chosen_KL_mean": -660.2740478515625, + "KL/mean": -862.5133666992188, + "KL/rejected_KL_mean": -1064.7528076171875, + "KL/std": 578.7081298828125, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0010660856496542692, + "fcm_dpo/delta": -0.05491678789258003, + "fcm_dpo/margin": 404.4787902832031, + "fcm_dpo/q_t": 0.4025976061820984, + "grad_norm": 38.45912551879883, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.9426896572113037, + "logits/rejected": -0.9390045404434204, + "logps/chosen": -707.8754272460938, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1152.037353515625, + "loss": 1.136, + "margin_dpo/margin_mean": 404.4787902832031, + "margin_dpo/margin_std": 729.6232299804688, + "step": 530 + }, + { + "KL/chosen_KL_mean": -751.2568359375, + "KL/mean": -909.684814453125, + "KL/rejected_KL_mean": -1068.11279296875, + "KL/std": 594.5966186523438, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.001148285111412406, + "fcm_dpo/delta": 0.05123718082904816, + "fcm_dpo/margin": 316.85595703125, + "fcm_dpo/q_t": 0.42305827140808105, + "grad_norm": 43.48565673828125, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.0459530353546143, + "logits/rejected": -1.0223256349563599, + "logps/chosen": -806.5028686523438, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1138.71875, + "loss": 1.1531, + "margin_dpo/margin_mean": 316.85595703125, + "margin_dpo/margin_std": 567.612060546875, + "step": 531 + }, + { + "KL/chosen_KL_mean": -734.84375, + "KL/mean": -924.059326171875, + "KL/rejected_KL_mean": -1113.27490234375, + "KL/std": 581.4156494140625, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0010310335783287883, + "fcm_dpo/delta": -0.008013017475605011, + "fcm_dpo/margin": 378.4312438964844, + "fcm_dpo/q_t": 0.41106435656547546, + "grad_norm": 68.37248229980469, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -0.9860376119613647, + "logits/rejected": -0.961615264415741, + "logps/chosen": -805.1297607421875, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1199.8662109375, + "loss": 1.1619, + "margin_dpo/margin_mean": 378.43121337890625, + "margin_dpo/margin_std": 688.8643798828125, + "step": 532 + }, + { + "KL/chosen_KL_mean": -660.6729125976562, + "KL/mean": -794.4534301757812, + "KL/rejected_KL_mean": -928.23388671875, + "KL/std": 490.07489013671875, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0011233469704166055, + "fcm_dpo/delta": 0.00041316391434520483, + "fcm_dpo/margin": 267.5610046386719, + "fcm_dpo/q_t": 0.4326951205730438, + "grad_norm": 41.96712875366211, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -1.028921127319336, + "logits/rejected": -1.0061641931533813, + "logps/chosen": -725.5280151367188, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1004.822021484375, + "loss": 1.2077, + "margin_dpo/margin_mean": 267.5610046386719, + "margin_dpo/margin_std": 582.5079345703125, + "step": 533 + }, + { + "KL/chosen_KL_mean": -676.6849365234375, + "KL/mean": -862.72900390625, + "KL/rejected_KL_mean": -1048.7730712890625, + "KL/std": 513.5587158203125, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0010465008672326803, + "fcm_dpo/delta": 0.0019133538007736206, + "fcm_dpo/margin": 372.08819580078125, + "fcm_dpo/q_t": 0.4103556275367737, + "grad_norm": 26.502838134765625, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -0.9815844297409058, + "logits/rejected": -0.9804989099502563, + "logps/chosen": -736.8043212890625, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1127.3165283203125, + "loss": 1.1194, + "margin_dpo/margin_mean": 372.08819580078125, + "margin_dpo/margin_std": 547.061279296875, + "step": 534 + }, + { + "KL/chosen_KL_mean": -587.0618286132812, + "KL/mean": -801.0487060546875, + "KL/rejected_KL_mean": -1015.0355224609375, + "KL/std": 525.9488525390625, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.001057377434335649, + "fcm_dpo/delta": -0.10026499629020691, + "fcm_dpo/margin": 427.9737854003906, + "fcm_dpo/q_t": 0.390084445476532, + "grad_norm": 25.19883155822754, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.9359257817268372, + "logits/rejected": -0.9411388039588928, + "logps/chosen": -641.39208984375, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1111.34326171875, + "loss": 1.0808, + "margin_dpo/margin_mean": 427.9737854003906, + "margin_dpo/margin_std": 552.81982421875, + "step": 535 + }, + { + "KL/chosen_KL_mean": -524.899658203125, + "KL/mean": -780.73291015625, + "KL/rejected_KL_mean": -1036.5660400390625, + "KL/std": 589.3612060546875, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0008447887958027422, + "fcm_dpo/delta": -0.05769486725330353, + "fcm_dpo/margin": 511.6664733886719, + "fcm_dpo/q_t": 0.3999031186103821, + "grad_norm": 24.389259338378906, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.9003403186798096, + "logits/rejected": -0.9327056407928467, + "logps/chosen": -571.980224609375, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1125.6639404296875, + "loss": 1.0767, + "margin_dpo/margin_mean": 511.66644287109375, + "margin_dpo/margin_std": 714.7777709960938, + "step": 536 + }, + { + "KL/chosen_KL_mean": -562.1661376953125, + "KL/mean": -730.231201171875, + "KL/rejected_KL_mean": -898.2962646484375, + "KL/std": 485.2312316894531, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0008401878294534981, + "fcm_dpo/delta": 0.021016618236899376, + "fcm_dpo/margin": 336.1300354003906, + "fcm_dpo/q_t": 0.4347127377986908, + "grad_norm": 33.1090202331543, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.9427402019500732, + "logits/rejected": -0.9209805727005005, + "logps/chosen": -619.9136962890625, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -968.734619140625, + "loss": 1.1829, + "margin_dpo/margin_mean": 336.1300354003906, + "margin_dpo/margin_std": 590.2587890625, + "step": 537 + }, + { + "KL/chosen_KL_mean": -686.517822265625, + "KL/mean": -878.772705078125, + "KL/rejected_KL_mean": -1071.027587890625, + "KL/std": 506.6947937011719, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0009338191011920571, + "fcm_dpo/delta": 0.064121313393116, + "fcm_dpo/margin": 384.5097351074219, + "fcm_dpo/q_t": 0.42180943489074707, + "grad_norm": 26.116352081298828, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.9635825753211975, + "logits/rejected": -0.9489821195602417, + "logps/chosen": -752.9337768554688, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1155.255615234375, + "loss": 1.1175, + "margin_dpo/margin_mean": 384.5097351074219, + "margin_dpo/margin_std": 529.1771240234375, + "step": 538 + }, + { + "KL/chosen_KL_mean": -581.152099609375, + "KL/mean": -763.506103515625, + "KL/rejected_KL_mean": -945.8601684570312, + "KL/std": 429.1192321777344, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0010440616169944406, + "fcm_dpo/delta": 0.02792687714099884, + "fcm_dpo/margin": 364.7080383300781, + "fcm_dpo/q_t": 0.4141731858253479, + "grad_norm": 33.319786071777344, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.9643697738647461, + "logits/rejected": -0.9533451795578003, + "logps/chosen": -639.6449584960938, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1037.714111328125, + "loss": 1.101, + "margin_dpo/margin_mean": 364.70806884765625, + "margin_dpo/margin_std": 481.9061279296875, + "step": 539 + }, + { + "KL/chosen_KL_mean": -593.5488891601562, + "KL/mean": -833.126708984375, + "KL/rejected_KL_mean": -1072.70458984375, + "KL/std": 541.5235595703125, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0009419465786777437, + "fcm_dpo/delta": -0.09322336316108704, + "fcm_dpo/margin": 479.15557861328125, + "fcm_dpo/q_t": 0.3929232060909271, + "grad_norm": 30.28093147277832, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.9041993618011475, + "logits/rejected": -0.9246673583984375, + "logps/chosen": -657.0314331054688, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1189.134521484375, + "loss": 1.0719, + "margin_dpo/margin_mean": 479.15557861328125, + "margin_dpo/margin_std": 642.5653686523438, + "step": 540 + }, + { + "KL/chosen_KL_mean": -716.245849609375, + "KL/mean": -844.7294921875, + "KL/rejected_KL_mean": -973.2132568359375, + "KL/std": 497.0907287597656, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0008998019620776176, + "fcm_dpo/delta": 0.02505185268819332, + "fcm_dpo/margin": 256.9674072265625, + "fcm_dpo/q_t": 0.44805556535720825, + "grad_norm": 39.39170455932617, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.0489933490753174, + "logits/rejected": -1.043041706085205, + "logps/chosen": -794.5261840820312, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1076.48681640625, + "loss": 1.241, + "margin_dpo/margin_mean": 256.9674377441406, + "margin_dpo/margin_std": 595.6408081054688, + "step": 541 + }, + { + "KL/chosen_KL_mean": -606.02099609375, + "KL/mean": -850.662841796875, + "KL/rejected_KL_mean": -1095.3046875, + "KL/std": 556.0362548828125, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.000812588375993073, + "fcm_dpo/delta": -0.015589557588100433, + "fcm_dpo/margin": 489.2837829589844, + "fcm_dpo/q_t": 0.40638577938079834, + "grad_norm": 43.08638000488281, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -1.0107741355895996, + "logits/rejected": -1.045440435409546, + "logps/chosen": -663.5059814453125, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1191.77978515625, + "loss": 1.1037, + "margin_dpo/margin_mean": 489.2837829589844, + "margin_dpo/margin_std": 664.2471313476562, + "step": 542 + }, + { + "KL/chosen_KL_mean": -686.9749145507812, + "KL/mean": -857.873046875, + "KL/rejected_KL_mean": -1028.7711181640625, + "KL/std": 652.3204956054688, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0009889693465083838, + "fcm_dpo/delta": 0.09416899085044861, + "fcm_dpo/margin": 341.7962646484375, + "fcm_dpo/q_t": 0.43743741512298584, + "grad_norm": 41.49184799194336, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.9908128976821899, + "logits/rejected": -1.023362398147583, + "logps/chosen": -747.5924072265625, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1134.3701171875, + "loss": 1.2031, + "margin_dpo/margin_mean": 341.7962646484375, + "margin_dpo/margin_std": 781.2564086914062, + "step": 543 + }, + { + "KL/chosen_KL_mean": -705.4442749023438, + "KL/mean": -904.148193359375, + "KL/rejected_KL_mean": -1102.85205078125, + "KL/std": 532.5914306640625, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0010228848550468683, + "fcm_dpo/delta": -0.011566650122404099, + "fcm_dpo/margin": 397.4078369140625, + "fcm_dpo/q_t": 0.40789809823036194, + "grad_norm": 28.750972747802734, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.9685473442077637, + "logits/rejected": -0.9889190196990967, + "logps/chosen": -765.0865478515625, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1203.8067626953125, + "loss": 1.1104, + "margin_dpo/margin_mean": 397.4078369140625, + "margin_dpo/margin_std": 614.56640625, + "step": 544 + }, + { + "KL/chosen_KL_mean": -649.4158935546875, + "KL/mean": -885.8656616210938, + "KL/rejected_KL_mean": -1122.3154296875, + "KL/std": 538.6456298828125, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0009213591692969203, + "fcm_dpo/delta": -0.06585489213466644, + "fcm_dpo/margin": 472.8994140625, + "fcm_dpo/q_t": 0.3961293399333954, + "grad_norm": 26.994535446166992, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.8998408913612366, + "logits/rejected": -0.9357277154922485, + "logps/chosen": -717.0645141601562, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1218.223388671875, + "loss": 1.0743, + "margin_dpo/margin_mean": 472.8994140625, + "margin_dpo/margin_std": 626.8919677734375, + "step": 545 + }, + { + "KL/chosen_KL_mean": -610.9400024414062, + "KL/mean": -767.9046630859375, + "KL/rejected_KL_mean": -924.8692626953125, + "KL/std": 476.8546142578125, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.0010576581116765738, + "fcm_dpo/delta": 0.0986643135547638, + "fcm_dpo/margin": 313.9293212890625, + "fcm_dpo/q_t": 0.4284512400627136, + "grad_norm": 36.017555236816406, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.9738471508026123, + "logits/rejected": -0.9590544104576111, + "logps/chosen": -661.6842041015625, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1006.7355346679688, + "loss": 1.1519, + "margin_dpo/margin_mean": 313.9293518066406, + "margin_dpo/margin_std": 505.626220703125, + "step": 546 + }, + { + "KL/chosen_KL_mean": -610.9646606445312, + "KL/mean": -821.750732421875, + "KL/rejected_KL_mean": -1032.536865234375, + "KL/std": 533.4896240234375, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.001024933997541666, + "fcm_dpo/delta": -0.056229621171951294, + "fcm_dpo/margin": 421.57220458984375, + "fcm_dpo/q_t": 0.3993460237979889, + "grad_norm": 39.7552490234375, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.9101555943489075, + "logits/rejected": -0.9197220802307129, + "logps/chosen": -684.65234375, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1123.29833984375, + "loss": 1.0807, + "margin_dpo/margin_mean": 421.57220458984375, + "margin_dpo/margin_std": 582.0226440429688, + "step": 547 + }, + { + "KL/chosen_KL_mean": -647.4964599609375, + "KL/mean": -836.765869140625, + "KL/rejected_KL_mean": -1026.035400390625, + "KL/std": 553.211181640625, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0010135022457689047, + "fcm_dpo/delta": 0.026592375710606575, + "fcm_dpo/margin": 378.53887939453125, + "fcm_dpo/q_t": 0.4186980724334717, + "grad_norm": 30.77254295349121, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.9787480235099792, + "logits/rejected": -0.9837968349456787, + "logps/chosen": -712.7427978515625, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1120.1534423828125, + "loss": 1.124, + "margin_dpo/margin_mean": 378.53887939453125, + "margin_dpo/margin_std": 625.75048828125, + "step": 548 + }, + { + "KL/chosen_KL_mean": -664.22216796875, + "KL/mean": -802.9415283203125, + "KL/rejected_KL_mean": -941.660888671875, + "KL/std": 461.15252685546875, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0010441699996590614, + "fcm_dpo/delta": 0.008397620171308517, + "fcm_dpo/margin": 277.43865966796875, + "fcm_dpo/q_t": 0.4350124001502991, + "grad_norm": 47.32408142089844, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -1.0104793310165405, + "logits/rejected": -1.0005792379379272, + "logps/chosen": -713.4345703125, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1015.5711669921875, + "loss": 1.1953, + "margin_dpo/margin_mean": 277.43865966796875, + "margin_dpo/margin_std": 547.7349243164062, + "step": 549 + }, + { + "KL/chosen_KL_mean": -645.0916748046875, + "KL/mean": -818.9498291015625, + "KL/rejected_KL_mean": -992.80810546875, + "KL/std": 506.2775573730469, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.0010925406822934747, + "fcm_dpo/delta": 0.03250662982463837, + "fcm_dpo/margin": 347.7164001464844, + "fcm_dpo/q_t": 0.41644904017448425, + "grad_norm": 42.81041717529297, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.9073216915130615, + "logits/rejected": -0.9299839735031128, + "logps/chosen": -701.8985595703125, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1087.933837890625, + "loss": 1.124, + "margin_dpo/margin_mean": 347.7164306640625, + "margin_dpo/margin_std": 549.0792846679688, + "step": 550 + }, + { + "KL/chosen_KL_mean": -558.291259765625, + "KL/mean": -833.9419555664062, + "KL/rejected_KL_mean": -1109.5927734375, + "KL/std": 536.7257080078125, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0008524754084646702, + "fcm_dpo/delta": -0.15517154335975647, + "fcm_dpo/margin": 551.3014526367188, + "fcm_dpo/q_t": 0.37387317419052124, + "grad_norm": 49.040679931640625, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.8993926048278809, + "logits/rejected": -0.954781174659729, + "logps/chosen": -617.3975830078125, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1221.2655029296875, + "loss": 1.0186, + "margin_dpo/margin_mean": 551.3014526367188, + "margin_dpo/margin_std": 538.538330078125, + "step": 551 + }, + { + "KL/chosen_KL_mean": -533.4443359375, + "KL/mean": -838.3875732421875, + "KL/rejected_KL_mean": -1143.330810546875, + "KL/std": 620.9227294921875, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0006882591405883431, + "fcm_dpo/delta": -0.06372275948524475, + "fcm_dpo/margin": 609.8865356445312, + "fcm_dpo/q_t": 0.3962107002735138, + "grad_norm": 35.4857063293457, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.9574573040008545, + "logits/rejected": -0.9972236156463623, + "logps/chosen": -595.7989501953125, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1247.892822265625, + "loss": 1.0688, + "margin_dpo/margin_mean": 609.8865356445312, + "margin_dpo/margin_std": 702.5086669921875, + "step": 552 + }, + { + "KL/chosen_KL_mean": -621.122802734375, + "KL/mean": -784.8905029296875, + "KL/rejected_KL_mean": -948.6583251953125, + "KL/std": 512.807861328125, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0009008180350065231, + "fcm_dpo/delta": 0.1481509506702423, + "fcm_dpo/margin": 327.53546142578125, + "fcm_dpo/q_t": 0.44127804040908813, + "grad_norm": 25.92827033996582, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.9558432698249817, + "logits/rejected": -0.9528769254684448, + "logps/chosen": -689.381591796875, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1046.7554931640625, + "loss": 1.1731, + "margin_dpo/margin_mean": 327.53546142578125, + "margin_dpo/margin_std": 559.0849609375, + "step": 553 + }, + { + "KL/chosen_KL_mean": -633.1609497070312, + "KL/mean": -847.7535400390625, + "KL/rejected_KL_mean": -1062.3460693359375, + "KL/std": 536.1700439453125, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0009824573062360287, + "fcm_dpo/delta": -0.04598114266991615, + "fcm_dpo/margin": 429.18511962890625, + "fcm_dpo/q_t": 0.4033704102039337, + "grad_norm": 57.81589126586914, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.9455916285514832, + "logits/rejected": -0.9259899258613586, + "logps/chosen": -701.108642578125, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1152.1287841796875, + "loss": 1.1217, + "margin_dpo/margin_mean": 429.18511962890625, + "margin_dpo/margin_std": 682.432861328125, + "step": 554 + }, + { + "KL/chosen_KL_mean": -634.130615234375, + "KL/mean": -891.5674438476562, + "KL/rejected_KL_mean": -1149.0042724609375, + "KL/std": 582.853515625, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.000833613914437592, + "fcm_dpo/delta": -0.051001857966184616, + "fcm_dpo/margin": 514.8737182617188, + "fcm_dpo/q_t": 0.40110278129577637, + "grad_norm": 41.96858215332031, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -0.9838435649871826, + "logits/rejected": -1.0397891998291016, + "logps/chosen": -687.4610595703125, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1257.483642578125, + "loss": 1.093, + "margin_dpo/margin_mean": 514.8736572265625, + "margin_dpo/margin_std": 755.9553833007812, + "step": 555 + }, + { + "KL/chosen_KL_mean": -619.962646484375, + "KL/mean": -767.575439453125, + "KL/rejected_KL_mean": -915.1883544921875, + "KL/std": 455.46258544921875, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0010035810992121696, + "fcm_dpo/delta": 0.14432799816131592, + "fcm_dpo/margin": 295.22576904296875, + "fcm_dpo/q_t": 0.4395366907119751, + "grad_norm": 32.15121841430664, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.9909765720367432, + "logits/rejected": -0.9854000210762024, + "logps/chosen": -678.6070556640625, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1016.5287475585938, + "loss": 1.163, + "margin_dpo/margin_mean": 295.22576904296875, + "margin_dpo/margin_std": 462.01824951171875, + "step": 556 + }, + { + "KL/chosen_KL_mean": -678.8939208984375, + "KL/mean": -913.603759765625, + "KL/rejected_KL_mean": -1148.3134765625, + "KL/std": 606.7928466796875, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0009204911766573787, + "fcm_dpo/delta": -0.06962708383798599, + "fcm_dpo/margin": 469.4195556640625, + "fcm_dpo/q_t": 0.39773887395858765, + "grad_norm": 51.71062088012695, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -0.9994686841964722, + "logits/rejected": -1.0355925559997559, + "logps/chosen": -746.734619140625, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1258.253173828125, + "loss": 1.0935, + "margin_dpo/margin_mean": 469.4195556640625, + "margin_dpo/margin_std": 688.9013671875, + "step": 557 + }, + { + "KL/chosen_KL_mean": -629.8007202148438, + "KL/mean": -937.6903076171875, + "KL/rejected_KL_mean": -1245.579833984375, + "KL/std": 600.0916748046875, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0007529731374233961, + "fcm_dpo/delta": -0.12873995304107666, + "fcm_dpo/margin": 615.7791748046875, + "fcm_dpo/q_t": 0.37874358892440796, + "grad_norm": 26.18462562561035, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.0540165901184082, + "logits/rejected": -1.0735430717468262, + "logps/chosen": -692.1689453125, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1347.740966796875, + "loss": 1.037, + "margin_dpo/margin_mean": 615.7791748046875, + "margin_dpo/margin_std": 673.7421264648438, + "step": 558 + }, + { + "KL/chosen_KL_mean": -701.1896362304688, + "KL/mean": -954.133544921875, + "KL/rejected_KL_mean": -1207.0775146484375, + "KL/std": 592.470458984375, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0007604295969940722, + "fcm_dpo/delta": 0.023233801126480103, + "fcm_dpo/margin": 505.8878479003906, + "fcm_dpo/q_t": 0.41252970695495605, + "grad_norm": 24.069955825805664, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.1368141174316406, + "logits/rejected": -1.1402065753936768, + "logps/chosen": -761.9420166015625, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1300.519775390625, + "loss": 1.0951, + "margin_dpo/margin_mean": 505.8878479003906, + "margin_dpo/margin_std": 648.712158203125, + "step": 559 + }, + { + "KL/chosen_KL_mean": -657.3939208984375, + "KL/mean": -828.5643310546875, + "KL/rejected_KL_mean": -999.7348022460938, + "KL/std": 511.27996826171875, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0008098391117528081, + "fcm_dpo/delta": 0.034772761166095734, + "fcm_dpo/margin": 342.3409118652344, + "fcm_dpo/q_t": 0.437608540058136, + "grad_norm": 26.729141235351562, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.9040641784667969, + "logits/rejected": -0.8838648796081543, + "logps/chosen": -715.4976806640625, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1079.72607421875, + "loss": 1.1829, + "margin_dpo/margin_mean": 342.3409423828125, + "margin_dpo/margin_std": 584.12548828125, + "step": 560 + }, + { + "KL/chosen_KL_mean": -759.172119140625, + "KL/mean": -920.302490234375, + "KL/rejected_KL_mean": -1081.432861328125, + "KL/std": 530.906494140625, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0009913041722029448, + "fcm_dpo/delta": 0.1052679792046547, + "fcm_dpo/margin": 322.26068115234375, + "fcm_dpo/q_t": 0.43175017833709717, + "grad_norm": 45.66500473022461, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.0488073825836182, + "logits/rejected": -1.0699024200439453, + "logps/chosen": -810.0850219726562, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1180.50146484375, + "loss": 1.1919, + "margin_dpo/margin_mean": 322.2607116699219, + "margin_dpo/margin_std": 664.1737060546875, + "step": 561 + }, + { + "KL/chosen_KL_mean": -713.2972412109375, + "KL/mean": -922.2352905273438, + "KL/rejected_KL_mean": -1131.1732177734375, + "KL/std": 585.4476928710938, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.000937954755499959, + "fcm_dpo/delta": 0.0012168213725090027, + "fcm_dpo/margin": 417.87603759765625, + "fcm_dpo/q_t": 0.4111784100532532, + "grad_norm": 32.701107025146484, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.0261307954788208, + "logits/rejected": -1.029954433441162, + "logps/chosen": -772.7616577148438, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1227.7158203125, + "loss": 1.1174, + "margin_dpo/margin_mean": 417.87603759765625, + "margin_dpo/margin_std": 627.3534545898438, + "step": 562 + }, + { + "KL/chosen_KL_mean": -804.9948120117188, + "KL/mean": -967.0625, + "KL/rejected_KL_mean": -1129.1302490234375, + "KL/std": 637.6392211914062, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0009516784339211881, + "fcm_dpo/delta": -0.052980802953243256, + "fcm_dpo/margin": 324.13543701171875, + "fcm_dpo/q_t": 0.42142850160598755, + "grad_norm": 52.949893951416016, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.0193402767181396, + "logits/rejected": -1.0229434967041016, + "logps/chosen": -865.60302734375, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1223.697998046875, + "loss": 1.2444, + "margin_dpo/margin_mean": 324.13543701171875, + "margin_dpo/margin_std": 768.7203369140625, + "step": 563 + }, + { + "KL/chosen_KL_mean": -731.5855712890625, + "KL/mean": -927.5992431640625, + "KL/rejected_KL_mean": -1123.6129150390625, + "KL/std": 567.4203491210938, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.000967438449151814, + "fcm_dpo/delta": 0.030362479388713837, + "fcm_dpo/margin": 392.0273132324219, + "fcm_dpo/q_t": 0.41628676652908325, + "grad_norm": 39.21650695800781, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.102457880973816, + "logits/rejected": -1.0930469036102295, + "logps/chosen": -805.8539428710938, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1216.85107421875, + "loss": 1.1161, + "margin_dpo/margin_mean": 392.0273132324219, + "margin_dpo/margin_std": 574.1221923828125, + "step": 564 + }, + { + "KL/chosen_KL_mean": -767.8232421875, + "KL/mean": -991.4110107421875, + "KL/rejected_KL_mean": -1214.998779296875, + "KL/std": 661.7870483398438, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0009264518739655614, + "fcm_dpo/delta": -0.024389147758483887, + "fcm_dpo/margin": 447.1756286621094, + "fcm_dpo/q_t": 0.4096825420856476, + "grad_norm": 40.5957145690918, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.025184154510498, + "logits/rejected": -1.0267276763916016, + "logps/chosen": -836.8431396484375, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1300.77783203125, + "loss": 1.1389, + "margin_dpo/margin_mean": 447.1756286621094, + "margin_dpo/margin_std": 811.365234375, + "step": 565 + }, + { + "KL/chosen_KL_mean": -710.8924560546875, + "KL/mean": -951.005615234375, + "KL/rejected_KL_mean": -1191.1187744140625, + "KL/std": 655.048828125, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0008984344312921166, + "fcm_dpo/delta": -0.06253774464130402, + "fcm_dpo/margin": 480.226318359375, + "fcm_dpo/q_t": 0.3989356458187103, + "grad_norm": 32.377769470214844, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -0.997527003288269, + "logits/rejected": -1.0165774822235107, + "logps/chosen": -777.4378051757812, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1294.988037109375, + "loss": 1.1279, + "margin_dpo/margin_mean": 480.226318359375, + "margin_dpo/margin_std": 777.289794921875, + "step": 566 + }, + { + "KL/chosen_KL_mean": -701.48583984375, + "KL/mean": -886.89306640625, + "KL/rejected_KL_mean": -1072.3001708984375, + "KL/std": 463.2765197753906, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.000920231977943331, + "fcm_dpo/delta": 0.0880487933754921, + "fcm_dpo/margin": 370.814453125, + "fcm_dpo/q_t": 0.42590731382369995, + "grad_norm": 27.92223358154297, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.1106657981872559, + "logits/rejected": -1.1169605255126953, + "logps/chosen": -754.3441162109375, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1157.671142578125, + "loss": 1.1274, + "margin_dpo/margin_mean": 370.814453125, + "margin_dpo/margin_std": 517.0704345703125, + "step": 567 + }, + { + "KL/chosen_KL_mean": -680.7457885742188, + "KL/mean": -934.899658203125, + "KL/rejected_KL_mean": -1189.053466796875, + "KL/std": 556.766845703125, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0008575194515287876, + "fcm_dpo/delta": -0.06648175418376923, + "fcm_dpo/margin": 508.3077087402344, + "fcm_dpo/q_t": 0.3927525281906128, + "grad_norm": 30.725919723510742, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -0.9947052001953125, + "logits/rejected": -1.032778024673462, + "logps/chosen": -725.9381713867188, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1278.14599609375, + "loss": 1.0558, + "margin_dpo/margin_mean": 508.3077392578125, + "margin_dpo/margin_std": 588.199951171875, + "step": 568 + }, + { + "KL/chosen_KL_mean": -766.2860717773438, + "KL/mean": -974.7943115234375, + "KL/rejected_KL_mean": -1183.3026123046875, + "KL/std": 686.7083740234375, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0009196175960823894, + "fcm_dpo/delta": 0.012853320688009262, + "fcm_dpo/margin": 417.0164794921875, + "fcm_dpo/q_t": 0.41270917654037476, + "grad_norm": 42.41626739501953, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.0407395362854004, + "logits/rejected": -1.0457051992416382, + "logps/chosen": -830.0066528320312, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1262.40576171875, + "loss": 1.1548, + "margin_dpo/margin_mean": 417.0164794921875, + "margin_dpo/margin_std": 728.3641357421875, + "step": 569 + }, + { + "KL/chosen_KL_mean": -706.3309326171875, + "KL/mean": -890.91357421875, + "KL/rejected_KL_mean": -1075.49609375, + "KL/std": 532.2640380859375, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0009554984280839562, + "fcm_dpo/delta": 0.07299959659576416, + "fcm_dpo/margin": 369.1651306152344, + "fcm_dpo/q_t": 0.424812376499176, + "grad_norm": 25.38414764404297, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.12546968460083, + "logits/rejected": -1.1144479513168335, + "logps/chosen": -767.945556640625, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1157.637939453125, + "loss": 1.1375, + "margin_dpo/margin_mean": 369.1651306152344, + "margin_dpo/margin_std": 582.8961181640625, + "step": 570 + }, + { + "KL/chosen_KL_mean": -729.112548828125, + "KL/mean": -953.21875, + "KL/rejected_KL_mean": -1177.324951171875, + "KL/std": 557.555908203125, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0009227694245055318, + "fcm_dpo/delta": -0.02847745269536972, + "fcm_dpo/margin": 448.21246337890625, + "fcm_dpo/q_t": 0.4045218527317047, + "grad_norm": 43.35622024536133, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -0.9972771406173706, + "logits/rejected": -1.0120426416397095, + "logps/chosen": -782.1666259765625, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1268.661865234375, + "loss": 1.1091, + "margin_dpo/margin_mean": 448.21246337890625, + "margin_dpo/margin_std": 690.9599609375, + "step": 571 + }, + { + "KL/chosen_KL_mean": -768.8928833007812, + "KL/mean": -1001.8563232421875, + "KL/rejected_KL_mean": -1234.8197021484375, + "KL/std": 665.1800537109375, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.000880242558196187, + "fcm_dpo/delta": -0.021999845281243324, + "fcm_dpo/margin": 465.9267883300781, + "fcm_dpo/q_t": 0.40960174798965454, + "grad_norm": 27.348243713378906, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.0507442951202393, + "logits/rejected": -1.080924153327942, + "logps/chosen": -817.3521728515625, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1318.376708984375, + "loss": 1.13, + "margin_dpo/margin_mean": 465.92681884765625, + "margin_dpo/margin_std": 805.666015625, + "step": 572 + }, + { + "KL/chosen_KL_mean": -752.9931030273438, + "KL/mean": -959.6439208984375, + "KL/rejected_KL_mean": -1166.294677734375, + "KL/std": 583.8150634765625, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.000941689417231828, + "fcm_dpo/delta": 0.016215384006500244, + "fcm_dpo/margin": 413.30157470703125, + "fcm_dpo/q_t": 0.4137089252471924, + "grad_norm": 29.199546813964844, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.084615707397461, + "logits/rejected": -1.095890760421753, + "logps/chosen": -804.61572265625, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1251.61962890625, + "loss": 1.1048, + "margin_dpo/margin_mean": 413.30157470703125, + "margin_dpo/margin_std": 595.2252197265625, + "step": 573 + }, + { + "KL/chosen_KL_mean": -693.208984375, + "KL/mean": -919.9398193359375, + "KL/rejected_KL_mean": -1146.670654296875, + "KL/std": 540.618896484375, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0009166327654384077, + "fcm_dpo/delta": -0.027354443445801735, + "fcm_dpo/margin": 453.4616394042969, + "fcm_dpo/q_t": 0.40286535024642944, + "grad_norm": 39.16676712036133, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.1048978567123413, + "logits/rejected": -1.1292717456817627, + "logps/chosen": -744.25341796875, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1239.47705078125, + "loss": 1.09, + "margin_dpo/margin_mean": 453.46160888671875, + "margin_dpo/margin_std": 616.9259643554688, + "step": 574 + }, + { + "KL/chosen_KL_mean": -759.01708984375, + "KL/mean": -962.711181640625, + "KL/rejected_KL_mean": -1166.4052734375, + "KL/std": 562.2581787109375, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0007753444951958954, + "fcm_dpo/delta": -0.06450725346803665, + "fcm_dpo/margin": 407.38818359375, + "fcm_dpo/q_t": 0.42131945490837097, + "grad_norm": 27.12775993347168, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.9311203360557556, + "logits/rejected": -0.9386662244796753, + "logps/chosen": -830.8072509765625, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1261.79150390625, + "loss": 1.1753, + "margin_dpo/margin_mean": 407.3881530761719, + "margin_dpo/margin_std": 729.4576416015625, + "step": 575 + }, + { + "KL/chosen_KL_mean": -730.3524169921875, + "KL/mean": -889.8451538085938, + "KL/rejected_KL_mean": -1049.3380126953125, + "KL/std": 508.9241943359375, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0008465542923659086, + "fcm_dpo/delta": 0.043933454900979996, + "fcm_dpo/margin": 318.985595703125, + "fcm_dpo/q_t": 0.4400612413883209, + "grad_norm": 28.7397403717041, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.0479196310043335, + "logits/rejected": -1.076101541519165, + "logps/chosen": -784.6153564453125, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1150.09228515625, + "loss": 1.1968, + "margin_dpo/margin_mean": 318.9855651855469, + "margin_dpo/margin_std": 597.7258911132812, + "step": 576 + }, + { + "KL/chosen_KL_mean": -645.6124877929688, + "KL/mean": -855.6819458007812, + "KL/rejected_KL_mean": -1065.75146484375, + "KL/std": 571.1446533203125, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0008858378860168159, + "fcm_dpo/delta": 0.04430598393082619, + "fcm_dpo/margin": 420.138916015625, + "fcm_dpo/q_t": 0.4191765785217285, + "grad_norm": 24.229158401489258, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.0603196620941162, + "logits/rejected": -1.0447897911071777, + "logps/chosen": -707.522216796875, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1149.8221435546875, + "loss": 1.1276, + "margin_dpo/margin_mean": 420.138916015625, + "margin_dpo/margin_std": 631.5208740234375, + "step": 577 + }, + { + "KL/chosen_KL_mean": -638.2508544921875, + "KL/mean": -869.6015014648438, + "KL/rejected_KL_mean": -1100.9521484375, + "KL/std": 571.8765258789062, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.0008566380711272359, + "fcm_dpo/delta": -0.007108829915523529, + "fcm_dpo/margin": 462.70135498046875, + "fcm_dpo/q_t": 0.40710175037384033, + "grad_norm": 31.410873413085938, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.9340611696243286, + "logits/rejected": -0.968732476234436, + "logps/chosen": -687.5145263671875, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1184.388427734375, + "loss": 1.0938, + "margin_dpo/margin_mean": 462.70135498046875, + "margin_dpo/margin_std": 587.1175537109375, + "step": 578 + }, + { + "KL/chosen_KL_mean": -741.0704345703125, + "KL/mean": -876.5369873046875, + "KL/rejected_KL_mean": -1012.0033569335938, + "KL/std": 570.01953125, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0010091912699863315, + "fcm_dpo/delta": 0.05066795274615288, + "fcm_dpo/margin": 270.932861328125, + "fcm_dpo/q_t": 0.44031286239624023, + "grad_norm": 65.59943389892578, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.9676034450531006, + "logits/rejected": -0.9647072553634644, + "logps/chosen": -799.956298828125, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1106.791015625, + "loss": 1.2276, + "margin_dpo/margin_mean": 270.93292236328125, + "margin_dpo/margin_std": 630.8173217773438, + "step": 579 + }, + { + "KL/chosen_KL_mean": -552.2589721679688, + "KL/mean": -812.1983642578125, + "KL/rejected_KL_mean": -1072.1376953125, + "KL/std": 636.0146484375, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0008385708788409829, + "fcm_dpo/delta": -0.08361184597015381, + "fcm_dpo/margin": 519.8788452148438, + "fcm_dpo/q_t": 0.3960821032524109, + "grad_norm": 34.06106185913086, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8970617055892944, + "logits/rejected": -0.9277414679527283, + "logps/chosen": -600.9658203125, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1153.896240234375, + "loss": 1.0825, + "margin_dpo/margin_mean": 519.8788452148438, + "margin_dpo/margin_std": 744.65966796875, + "step": 580 + }, + { + "KL/chosen_KL_mean": -730.3821411132812, + "KL/mean": -920.6229858398438, + "KL/rejected_KL_mean": -1110.86376953125, + "KL/std": 590.761962890625, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.0009422561852261424, + "fcm_dpo/delta": 0.06384095549583435, + "fcm_dpo/margin": 380.4817199707031, + "fcm_dpo/q_t": 0.4236143231391907, + "grad_norm": 37.83233642578125, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.045546054840088, + "logits/rejected": -1.0597128868103027, + "logps/chosen": -792.6044921875, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1202.599609375, + "loss": 1.1464, + "margin_dpo/margin_mean": 380.48175048828125, + "margin_dpo/margin_std": 654.8671875, + "step": 581 + }, + { + "KL/chosen_KL_mean": -663.98291015625, + "KL/mean": -884.2652587890625, + "KL/rejected_KL_mean": -1104.547607421875, + "KL/std": 535.1427001953125, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0009837101679295301, + "fcm_dpo/delta": -0.08480895310640335, + "fcm_dpo/margin": 440.5647277832031, + "fcm_dpo/q_t": 0.3922159671783447, + "grad_norm": 33.73926544189453, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -1.0071823596954346, + "logits/rejected": -1.0349555015563965, + "logps/chosen": -722.2115478515625, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1214.6171875, + "loss": 1.1097, + "margin_dpo/margin_mean": 440.56475830078125, + "margin_dpo/margin_std": 578.998046875, + "step": 582 + }, + { + "KL/chosen_KL_mean": -755.7252197265625, + "KL/mean": -893.9094848632812, + "KL/rejected_KL_mean": -1032.0936279296875, + "KL/std": 595.7422485351562, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0008164974860846996, + "fcm_dpo/delta": -0.0013605619315057993, + "fcm_dpo/margin": 276.36834716796875, + "fcm_dpo/q_t": 0.4494253695011139, + "grad_norm": 38.58605194091797, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.9783551096916199, + "logits/rejected": -0.9746303558349609, + "logps/chosen": -812.588134765625, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1120.49755859375, + "loss": 1.2611, + "margin_dpo/margin_mean": 276.36834716796875, + "margin_dpo/margin_std": 718.41357421875, + "step": 583 + }, + { + "KL/chosen_KL_mean": -633.4464721679688, + "KL/mean": -902.180908203125, + "KL/rejected_KL_mean": -1170.915283203125, + "KL/std": 608.067626953125, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0007747331401333213, + "fcm_dpo/delta": -0.028918039053678513, + "fcm_dpo/margin": 537.46875, + "fcm_dpo/q_t": 0.4025737941265106, + "grad_norm": 19.63285255432129, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -0.986220121383667, + "logits/rejected": -1.0069942474365234, + "logps/chosen": -690.34716796875, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1268.55126953125, + "loss": 1.0713, + "margin_dpo/margin_mean": 537.46875, + "margin_dpo/margin_std": 643.06298828125, + "step": 584 + }, + { + "KL/chosen_KL_mean": -779.0941772460938, + "KL/mean": -1014.2800903320312, + "KL/rejected_KL_mean": -1249.466064453125, + "KL/std": 677.1864013671875, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0007966947159729898, + "fcm_dpo/delta": 0.03990716487169266, + "fcm_dpo/margin": 470.371826171875, + "fcm_dpo/q_t": 0.4176916182041168, + "grad_norm": 31.820642471313477, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.07651948928833, + "logits/rejected": -1.104307770729065, + "logps/chosen": -840.068115234375, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1333.635498046875, + "loss": 1.1456, + "margin_dpo/margin_mean": 470.371826171875, + "margin_dpo/margin_std": 820.4464111328125, + "step": 585 + }, + { + "KL/chosen_KL_mean": -788.8521728515625, + "KL/mean": -943.93310546875, + "KL/rejected_KL_mean": -1099.013916015625, + "KL/std": 580.5469970703125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0010007360251620412, + "fcm_dpo/delta": 0.1305122673511505, + "fcm_dpo/margin": 310.16180419921875, + "fcm_dpo/q_t": 0.43785321712493896, + "grad_norm": 55.22218322753906, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.0562876462936401, + "logits/rejected": -1.0779566764831543, + "logps/chosen": -845.707763671875, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1190.816650390625, + "loss": 1.1985, + "margin_dpo/margin_mean": 310.16180419921875, + "margin_dpo/margin_std": 638.7469482421875, + "step": 586 + }, + { + "KL/chosen_KL_mean": -554.4176635742188, + "KL/mean": -765.8219604492188, + "KL/rejected_KL_mean": -977.2261962890625, + "KL/std": 601.9225463867188, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0010241747368127108, + "fcm_dpo/delta": -0.05821375921368599, + "fcm_dpo/margin": 422.8085632324219, + "fcm_dpo/q_t": 0.39820683002471924, + "grad_norm": 31.542490005493164, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.0706478357315063, + "logits/rejected": -1.0984125137329102, + "logps/chosen": -599.1092529296875, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1059.85009765625, + "loss": 1.0901, + "margin_dpo/margin_mean": 422.80853271484375, + "margin_dpo/margin_std": 611.4573974609375, + "step": 587 + }, + { + "KL/chosen_KL_mean": -721.3621826171875, + "KL/mean": -904.2818603515625, + "KL/rejected_KL_mean": -1087.2015380859375, + "KL/std": 524.9571533203125, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0009967966470867395, + "fcm_dpo/delta": 0.05296815186738968, + "fcm_dpo/margin": 365.8393859863281, + "fcm_dpo/q_t": 0.41959983110427856, + "grad_norm": 32.92317581176758, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.0092002153396606, + "logits/rejected": -1.0338335037231445, + "logps/chosen": -771.6571044921875, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1194.571533203125, + "loss": 1.1352, + "margin_dpo/margin_mean": 365.83941650390625, + "margin_dpo/margin_std": 559.2975463867188, + "step": 588 + }, + { + "KL/chosen_KL_mean": -717.5643920898438, + "KL/mean": -957.2388916015625, + "KL/rejected_KL_mean": -1196.9134521484375, + "KL/std": 576.5167846679688, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0009336763177998364, + "fcm_dpo/delta": -0.08870557695627213, + "fcm_dpo/margin": 479.34906005859375, + "fcm_dpo/q_t": 0.39069461822509766, + "grad_norm": 31.190114974975586, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -0.9594367146492004, + "logits/rejected": -0.968762993812561, + "logps/chosen": -777.4942626953125, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1308.56884765625, + "loss": 1.0654, + "margin_dpo/margin_mean": 479.34906005859375, + "margin_dpo/margin_std": 618.2476806640625, + "step": 589 + }, + { + "KL/chosen_KL_mean": -610.7032470703125, + "KL/mean": -851.4964599609375, + "KL/rejected_KL_mean": -1092.28955078125, + "KL/std": 560.0525512695312, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0008607573690824211, + "fcm_dpo/delta": -0.025180064141750336, + "fcm_dpo/margin": 481.5863037109375, + "fcm_dpo/q_t": 0.40354564785957336, + "grad_norm": 23.0913028717041, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.9822874069213867, + "logits/rejected": -1.0102424621582031, + "logps/chosen": -666.5130615234375, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1198.3524169921875, + "loss": 1.0802, + "margin_dpo/margin_mean": 481.5863037109375, + "margin_dpo/margin_std": 624.2484741210938, + "step": 590 + }, + { + "KL/chosen_KL_mean": -666.4302978515625, + "KL/mean": -876.166259765625, + "KL/rejected_KL_mean": -1085.902099609375, + "KL/std": 542.080322265625, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0008978393743745983, + "fcm_dpo/delta": 0.03757140040397644, + "fcm_dpo/margin": 419.4718017578125, + "fcm_dpo/q_t": 0.41537439823150635, + "grad_norm": 31.897592544555664, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.0660505294799805, + "logits/rejected": -1.0602378845214844, + "logps/chosen": -722.6708984375, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1169.6884765625, + "loss": 1.1202, + "margin_dpo/margin_mean": 419.4718017578125, + "margin_dpo/margin_std": 628.8763427734375, + "step": 591 + }, + { + "KL/chosen_KL_mean": -694.1058349609375, + "KL/mean": -893.2886962890625, + "KL/rejected_KL_mean": -1092.4716796875, + "KL/std": 556.25927734375, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0009542823536321521, + "fcm_dpo/delta": 0.03209465742111206, + "fcm_dpo/margin": 398.3658447265625, + "fcm_dpo/q_t": 0.4169124960899353, + "grad_norm": 41.34512710571289, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.0209119319915771, + "logits/rejected": -1.0272916555404663, + "logps/chosen": -742.0460815429688, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1168.20458984375, + "loss": 1.1721, + "margin_dpo/margin_mean": 398.3658447265625, + "margin_dpo/margin_std": 758.87109375, + "step": 592 + }, + { + "KL/chosen_KL_mean": -679.8154296875, + "KL/mean": -824.459228515625, + "KL/rejected_KL_mean": -969.10302734375, + "KL/std": 587.4564208984375, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.001031426596455276, + "fcm_dpo/delta": 0.024800950661301613, + "fcm_dpo/margin": 289.28753662109375, + "fcm_dpo/q_t": 0.4358917474746704, + "grad_norm": 62.813846588134766, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.9074722528457642, + "logits/rejected": -0.9004065990447998, + "logps/chosen": -728.5061645507812, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1030.010986328125, + "loss": 1.2174, + "margin_dpo/margin_mean": 289.28753662109375, + "margin_dpo/margin_std": 654.1934204101562, + "step": 593 + }, + { + "KL/chosen_KL_mean": -626.3131103515625, + "KL/mean": -814.7869873046875, + "KL/rejected_KL_mean": -1003.2608642578125, + "KL/std": 552.5802001953125, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.0010870166588574648, + "fcm_dpo/delta": -0.03803686052560806, + "fcm_dpo/margin": 376.94769287109375, + "fcm_dpo/q_t": 0.4029105305671692, + "grad_norm": 54.68209457397461, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.0033159255981445, + "logits/rejected": -0.9932465553283691, + "logps/chosen": -681.248046875, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1089.360595703125, + "loss": 1.1364, + "margin_dpo/margin_mean": 376.94769287109375, + "margin_dpo/margin_std": 580.914794921875, + "step": 594 + }, + { + "KL/chosen_KL_mean": -590.2716674804688, + "KL/mean": -798.0963134765625, + "KL/rejected_KL_mean": -1005.9209594726562, + "KL/std": 527.621826171875, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0009453489328734577, + "fcm_dpo/delta": 0.01016305387020111, + "fcm_dpo/margin": 415.6493225097656, + "fcm_dpo/q_t": 0.41294392943382263, + "grad_norm": 53.49211120605469, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.970264196395874, + "logits/rejected": -0.9880306124687195, + "logps/chosen": -639.6920776367188, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1086.54833984375, + "loss": 1.1058, + "margin_dpo/margin_mean": 415.6493225097656, + "margin_dpo/margin_std": 597.9583129882812, + "step": 595 + }, + { + "KL/chosen_KL_mean": -709.765625, + "KL/mean": -855.121337890625, + "KL/rejected_KL_mean": -1000.47705078125, + "KL/std": 552.8822021484375, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0011489235330373049, + "fcm_dpo/delta": 0.09330044686794281, + "fcm_dpo/margin": 290.71142578125, + "fcm_dpo/q_t": 0.4343324303627014, + "grad_norm": 93.66545867919922, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.9842763543128967, + "logits/rejected": -0.9520334005355835, + "logps/chosen": -769.5573120117188, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1080.568115234375, + "loss": 1.2066, + "margin_dpo/margin_mean": 290.71142578125, + "margin_dpo/margin_std": 659.2149658203125, + "step": 596 + }, + { + "KL/chosen_KL_mean": -605.57763671875, + "KL/mean": -864.177001953125, + "KL/rejected_KL_mean": -1122.7763671875, + "KL/std": 595.9703369140625, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0008297483436763287, + "fcm_dpo/delta": -0.1401767134666443, + "fcm_dpo/margin": 517.19873046875, + "fcm_dpo/q_t": 0.3865550458431244, + "grad_norm": 30.938121795654297, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.8776600956916809, + "logits/rejected": -0.9235655069351196, + "logps/chosen": -662.83837890625, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1223.469970703125, + "loss": 1.0834, + "margin_dpo/margin_mean": 517.1986694335938, + "margin_dpo/margin_std": 668.316162109375, + "step": 597 + }, + { + "KL/chosen_KL_mean": -645.602294921875, + "KL/mean": -850.23193359375, + "KL/rejected_KL_mean": -1054.861572265625, + "KL/std": 531.68505859375, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0008029807358980179, + "fcm_dpo/delta": -0.050433676689863205, + "fcm_dpo/margin": 409.2593994140625, + "fcm_dpo/q_t": 0.41770249605178833, + "grad_norm": 26.686721801757812, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -0.9931685328483582, + "logits/rejected": -1.0055735111236572, + "logps/chosen": -698.1207885742188, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1144.3055419921875, + "loss": 1.1418, + "margin_dpo/margin_mean": 409.2593994140625, + "margin_dpo/margin_std": 592.2933349609375, + "step": 598 + }, + { + "KL/chosen_KL_mean": -626.4500732421875, + "KL/mean": -800.4187622070312, + "KL/rejected_KL_mean": -974.387451171875, + "KL/std": 483.94842529296875, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0007874942384660244, + "fcm_dpo/delta": -0.009737352840602398, + "fcm_dpo/margin": 347.93743896484375, + "fcm_dpo/q_t": 0.4326390027999878, + "grad_norm": 21.70595932006836, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.8962666988372803, + "logits/rejected": -0.9054356813430786, + "logps/chosen": -676.2527465820312, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1057.365966796875, + "loss": 1.168, + "margin_dpo/margin_mean": 347.93743896484375, + "margin_dpo/margin_std": 467.16839599609375, + "step": 599 + }, + { + "KL/chosen_KL_mean": -702.6114501953125, + "KL/mean": -863.553955078125, + "KL/rejected_KL_mean": -1024.496337890625, + "KL/std": 520.333251953125, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.0009568602545186877, + "fcm_dpo/delta": 0.1319303810596466, + "fcm_dpo/margin": 321.88494873046875, + "fcm_dpo/q_t": 0.43882617354393005, + "grad_norm": 38.444881439208984, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.0397906303405762, + "logits/rejected": -1.0333709716796875, + "logps/chosen": -769.04638671875, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1109.952880859375, + "loss": 1.1911, + "margin_dpo/margin_mean": 321.88494873046875, + "margin_dpo/margin_std": 643.2849731445312, + "step": 600 + }, + { + "KL/chosen_KL_mean": -702.807373046875, + "KL/mean": -912.3177490234375, + "KL/rejected_KL_mean": -1121.828125, + "KL/std": 553.8662109375, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0009801883716136217, + "fcm_dpo/delta": -0.01944911666214466, + "fcm_dpo/margin": 419.02081298828125, + "fcm_dpo/q_t": 0.40293174982070923, + "grad_norm": 32.09441375732422, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -0.9758914113044739, + "logits/rejected": -0.9933589696884155, + "logps/chosen": -761.9410400390625, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1216.5191650390625, + "loss": 1.0856, + "margin_dpo/margin_mean": 419.02081298828125, + "margin_dpo/margin_std": 550.6053466796875, + "step": 601 + }, + { + "KL/chosen_KL_mean": -479.7313232421875, + "KL/mean": -729.894287109375, + "KL/rejected_KL_mean": -980.057373046875, + "KL/std": 495.54962158203125, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0009083700715564191, + "fcm_dpo/delta": -0.10118807107210159, + "fcm_dpo/margin": 500.3260192871094, + "fcm_dpo/q_t": 0.3865256905555725, + "grad_norm": 69.2566909790039, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -0.9927771091461182, + "logits/rejected": -1.0256961584091187, + "logps/chosen": -528.3248291015625, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1067.725830078125, + "loss": 1.0464, + "margin_dpo/margin_mean": 500.3260192871094, + "margin_dpo/margin_std": 536.3623046875, + "step": 602 + }, + { + "KL/chosen_KL_mean": -610.1720581054688, + "KL/mean": -848.7678833007812, + "KL/rejected_KL_mean": -1087.3636474609375, + "KL/std": 582.8988037109375, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0008255116990767419, + "fcm_dpo/delta": 0.009567895904183388, + "fcm_dpo/margin": 477.191650390625, + "fcm_dpo/q_t": 0.41279083490371704, + "grad_norm": 32.93044662475586, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -0.9903879165649414, + "logits/rejected": -0.9831830263137817, + "logps/chosen": -680.586669921875, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1187.689208984375, + "loss": 1.1056, + "margin_dpo/margin_mean": 477.191650390625, + "margin_dpo/margin_std": 704.5848388671875, + "step": 603 + }, + { + "KL/chosen_KL_mean": -604.884521484375, + "KL/mean": -850.2882080078125, + "KL/rejected_KL_mean": -1095.69189453125, + "KL/std": 557.6204833984375, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0008146062027662992, + "fcm_dpo/delta": 0.0001351330429315567, + "fcm_dpo/margin": 490.8074035644531, + "fcm_dpo/q_t": 0.40916040539741516, + "grad_norm": 38.27802658081055, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.9702785015106201, + "logits/rejected": -1.004528284072876, + "logps/chosen": -651.3425903320312, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1187.54638671875, + "loss": 1.1004, + "margin_dpo/margin_mean": 490.8074035644531, + "margin_dpo/margin_std": 693.0052490234375, + "step": 604 + }, + { + "KL/chosen_KL_mean": -633.1536865234375, + "KL/mean": -830.642578125, + "KL/rejected_KL_mean": -1028.1314697265625, + "KL/std": 524.9966430664062, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0008949829498305917, + "fcm_dpo/delta": 0.07238632440567017, + "fcm_dpo/margin": 394.977783203125, + "fcm_dpo/q_t": 0.42492377758026123, + "grad_norm": 38.80448913574219, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.9616391658782959, + "logits/rejected": -0.9820314645767212, + "logps/chosen": -699.4030151367188, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1130.4365234375, + "loss": 1.1385, + "margin_dpo/margin_mean": 394.977783203125, + "margin_dpo/margin_std": 627.3665771484375, + "step": 605 + }, + { + "KL/chosen_KL_mean": -626.211181640625, + "KL/mean": -838.0142211914062, + "KL/rejected_KL_mean": -1049.8173828125, + "KL/std": 531.291748046875, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0009638095507398248, + "fcm_dpo/delta": -0.017124518752098083, + "fcm_dpo/margin": 423.60614013671875, + "fcm_dpo/q_t": 0.4059169888496399, + "grad_norm": 26.27326011657715, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.9585670232772827, + "logits/rejected": -0.9715864658355713, + "logps/chosen": -681.0302734375, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1148.188720703125, + "loss": 1.0882, + "margin_dpo/margin_mean": 423.60614013671875, + "margin_dpo/margin_std": 562.107666015625, + "step": 606 + }, + { + "KL/chosen_KL_mean": -664.5488891601562, + "KL/mean": -828.581298828125, + "KL/rejected_KL_mean": -992.6136474609375, + "KL/std": 521.07080078125, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0010647273156791925, + "fcm_dpo/delta": 0.06575503945350647, + "fcm_dpo/margin": 328.06475830078125, + "fcm_dpo/q_t": 0.42434990406036377, + "grad_norm": 34.5279655456543, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.9723426103591919, + "logits/rejected": -0.9582047462463379, + "logps/chosen": -722.6329345703125, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1072.3907470703125, + "loss": 1.1543, + "margin_dpo/margin_mean": 328.06475830078125, + "margin_dpo/margin_std": 572.1151123046875, + "step": 607 + }, + { + "KL/chosen_KL_mean": -599.10595703125, + "KL/mean": -798.927978515625, + "KL/rejected_KL_mean": -998.75, + "KL/std": 486.12176513671875, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.001036277157254517, + "fcm_dpo/delta": -0.02508939430117607, + "fcm_dpo/margin": 399.6440734863281, + "fcm_dpo/q_t": 0.4025970697402954, + "grad_norm": 39.96210479736328, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.0201570987701416, + "logits/rejected": -1.0304514169692993, + "logps/chosen": -656.5568237304688, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1093.5234375, + "loss": 1.0788, + "margin_dpo/margin_mean": 399.64404296875, + "margin_dpo/margin_std": 502.84564208984375, + "step": 608 + }, + { + "KL/chosen_KL_mean": -625.5040283203125, + "KL/mean": -888.441162109375, + "KL/rejected_KL_mean": -1151.378173828125, + "KL/std": 666.3944091796875, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.0008594167302362621, + "fcm_dpo/delta": -0.09702034294605255, + "fcm_dpo/margin": 525.8741455078125, + "fcm_dpo/q_t": 0.3949633240699768, + "grad_norm": 29.249309539794922, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.0417685508728027, + "logits/rejected": -1.0658833980560303, + "logps/chosen": -684.3094482421875, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1240.1943359375, + "loss": 1.0786, + "margin_dpo/margin_mean": 525.8742065429688, + "margin_dpo/margin_std": 787.166748046875, + "step": 609 + }, + { + "KL/chosen_KL_mean": -631.95361328125, + "KL/mean": -791.3546142578125, + "KL/rejected_KL_mean": -950.7557373046875, + "KL/std": 494.2914733886719, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0009960741735994816, + "fcm_dpo/delta": 0.11386167258024216, + "fcm_dpo/margin": 318.8021240234375, + "fcm_dpo/q_t": 0.4325554370880127, + "grad_norm": 40.697330474853516, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.9712044596672058, + "logits/rejected": -0.9452321529388428, + "logps/chosen": -697.648681640625, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1034.1611328125, + "loss": 1.1692, + "margin_dpo/margin_mean": 318.8021240234375, + "margin_dpo/margin_std": 572.7630615234375, + "step": 610 + }, + { + "KL/chosen_KL_mean": -635.2578125, + "KL/mean": -909.711181640625, + "KL/rejected_KL_mean": -1184.1644287109375, + "KL/std": 663.8045654296875, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0008432117174379528, + "fcm_dpo/delta": -0.12401486933231354, + "fcm_dpo/margin": 548.9066162109375, + "fcm_dpo/q_t": 0.38857901096343994, + "grad_norm": 30.683746337890625, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.0351388454437256, + "logits/rejected": -1.0823793411254883, + "logps/chosen": -687.8572998046875, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1270.495361328125, + "loss": 1.0702, + "margin_dpo/margin_mean": 548.9066162109375, + "margin_dpo/margin_std": 768.596923828125, + "step": 611 + }, + { + "KL/chosen_KL_mean": -690.9043579101562, + "KL/mean": -900.0711669921875, + "KL/rejected_KL_mean": -1109.238037109375, + "KL/std": 532.022705078125, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0008869701996445656, + "fcm_dpo/delta": 0.03892756253480911, + "fcm_dpo/margin": 418.3336181640625, + "fcm_dpo/q_t": 0.4165932536125183, + "grad_norm": 28.735475540161133, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -1.0041875839233398, + "logits/rejected": -1.0075181722640991, + "logps/chosen": -750.2280883789062, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1197.55029296875, + "loss": 1.1135, + "margin_dpo/margin_mean": 418.3336181640625, + "margin_dpo/margin_std": 586.0115356445312, + "step": 612 + }, + { + "KL/chosen_KL_mean": -596.6041259765625, + "KL/mean": -865.2132568359375, + "KL/rejected_KL_mean": -1133.8223876953125, + "KL/std": 586.6395874023438, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0007895183516666293, + "fcm_dpo/delta": -0.046130187809467316, + "fcm_dpo/margin": 537.21826171875, + "fcm_dpo/q_t": 0.3976578116416931, + "grad_norm": 28.2447509765625, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -1.0270159244537354, + "logits/rejected": -1.0836434364318848, + "logps/chosen": -656.3341064453125, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1238.929931640625, + "loss": 1.0612, + "margin_dpo/margin_mean": 537.21826171875, + "margin_dpo/margin_std": 617.8411865234375, + "step": 613 + }, + { + "KL/chosen_KL_mean": -683.5602416992188, + "KL/mean": -936.4262084960938, + "KL/rejected_KL_mean": -1189.292236328125, + "KL/std": 602.0802001953125, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0007989507867023349, + "fcm_dpo/delta": -0.007680162787437439, + "fcm_dpo/margin": 505.73199462890625, + "fcm_dpo/q_t": 0.40751612186431885, + "grad_norm": 35.26876449584961, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.9546043276786804, + "logits/rejected": -1.0301423072814941, + "logps/chosen": -736.4992065429688, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1293.9715576171875, + "loss": 1.101, + "margin_dpo/margin_mean": 505.73199462890625, + "margin_dpo/margin_std": 729.1088256835938, + "step": 614 + }, + { + "KL/chosen_KL_mean": -640.28759765625, + "KL/mean": -844.96484375, + "KL/rejected_KL_mean": -1049.6419677734375, + "KL/std": 611.680419921875, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.000880036037415266, + "fcm_dpo/delta": 0.05419084057211876, + "fcm_dpo/margin": 409.35443115234375, + "fcm_dpo/q_t": 0.4209446310997009, + "grad_norm": 33.03697204589844, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.9839037656784058, + "logits/rejected": -1.0060296058654785, + "logps/chosen": -706.1048583984375, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1144.8194580078125, + "loss": 1.1391, + "margin_dpo/margin_mean": 409.35443115234375, + "margin_dpo/margin_std": 675.0528564453125, + "step": 615 + }, + { + "KL/chosen_KL_mean": -780.3349609375, + "KL/mean": -949.9927368164062, + "KL/rejected_KL_mean": -1119.6505126953125, + "KL/std": 530.0982055664062, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0010057191830128431, + "fcm_dpo/delta": 0.08487111330032349, + "fcm_dpo/margin": 339.3155517578125, + "fcm_dpo/q_t": 0.4275793731212616, + "grad_norm": 35.978885650634766, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.0499340295791626, + "logits/rejected": -1.0310404300689697, + "logps/chosen": -845.4678344726562, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1194.35107421875, + "loss": 1.1523, + "margin_dpo/margin_mean": 339.3155517578125, + "margin_dpo/margin_std": 571.7545166015625, + "step": 616 + }, + { + "KL/chosen_KL_mean": -708.6893310546875, + "KL/mean": -836.7636108398438, + "KL/rejected_KL_mean": -964.837890625, + "KL/std": 481.6317443847656, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0012495677219703794, + "fcm_dpo/delta": 0.1090506836771965, + "fcm_dpo/margin": 256.1485595703125, + "fcm_dpo/q_t": 0.4353943467140198, + "grad_norm": 90.08831024169922, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.9055740833282471, + "logits/rejected": -0.8715794086456299, + "logps/chosen": -771.6948852539062, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1029.072021484375, + "loss": 1.208, + "margin_dpo/margin_mean": 256.1485290527344, + "margin_dpo/margin_std": 568.8765869140625, + "step": 617 + }, + { + "KL/chosen_KL_mean": -740.5645141601562, + "KL/mean": -1015.2177124023438, + "KL/rejected_KL_mean": -1289.870849609375, + "KL/std": 659.4688720703125, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0009442686568945646, + "fcm_dpo/delta": -0.24364520609378815, + "fcm_dpo/margin": 549.306396484375, + "fcm_dpo/q_t": 0.3701603412628174, + "grad_norm": 38.955535888671875, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -1.0169545412063599, + "logits/rejected": -1.0575779676437378, + "logps/chosen": -807.6658935546875, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1382.0242919921875, + "loss": 1.0903, + "margin_dpo/margin_mean": 549.306396484375, + "margin_dpo/margin_std": 852.4629516601562, + "step": 618 + }, + { + "KL/chosen_KL_mean": -735.8086547851562, + "KL/mean": -945.514892578125, + "KL/rejected_KL_mean": -1155.22119140625, + "KL/std": 635.7939453125, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0008637189166620374, + "fcm_dpo/delta": 0.05800282955169678, + "fcm_dpo/margin": 419.4125671386719, + "fcm_dpo/q_t": 0.4264986217021942, + "grad_norm": 39.2636604309082, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.0181564092636108, + "logits/rejected": -1.0320208072662354, + "logps/chosen": -791.786865234375, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1248.4066162109375, + "loss": 1.1748, + "margin_dpo/margin_mean": 419.41259765625, + "margin_dpo/margin_std": 845.4202880859375, + "step": 619 + }, + { + "KL/chosen_KL_mean": -677.5810546875, + "KL/mean": -869.0966796875, + "KL/rejected_KL_mean": -1060.6123046875, + "KL/std": 539.739990234375, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0009379057446494699, + "fcm_dpo/delta": 0.06312977522611618, + "fcm_dpo/margin": 383.0312194824219, + "fcm_dpo/q_t": 0.4227410554885864, + "grad_norm": 29.400907516479492, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.0447208881378174, + "logits/rejected": -1.05232572555542, + "logps/chosen": -737.3785400390625, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1139.02294921875, + "loss": 1.1375, + "margin_dpo/margin_mean": 383.0312194824219, + "margin_dpo/margin_std": 603.0001220703125, + "step": 620 + }, + { + "KL/chosen_KL_mean": -654.668701171875, + "KL/mean": -947.2188110351562, + "KL/rejected_KL_mean": -1239.768798828125, + "KL/std": 664.20166015625, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0008394881151616573, + "fcm_dpo/delta": -0.17594929039478302, + "fcm_dpo/margin": 585.100341796875, + "fcm_dpo/q_t": 0.37430325150489807, + "grad_norm": 41.29154968261719, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.0949594974517822, + "logits/rejected": -1.1471253633499146, + "logps/chosen": -708.6024169921875, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1328.138427734375, + "loss": 1.0593, + "margin_dpo/margin_mean": 585.1002807617188, + "margin_dpo/margin_std": 730.66455078125, + "step": 621 + }, + { + "KL/chosen_KL_mean": -661.12158203125, + "KL/mean": -863.3720703125, + "KL/rejected_KL_mean": -1065.6224365234375, + "KL/std": 502.6478271484375, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0008088588365353644, + "fcm_dpo/delta": 0.10930877178907394, + "fcm_dpo/margin": 404.5008544921875, + "fcm_dpo/q_t": 0.4323442876338959, + "grad_norm": 22.646230697631836, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.941389799118042, + "logits/rejected": -0.9269174337387085, + "logps/chosen": -721.407470703125, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1151.1412353515625, + "loss": 1.1511, + "margin_dpo/margin_mean": 404.5008544921875, + "margin_dpo/margin_std": 631.4569091796875, + "step": 622 + }, + { + "KL/chosen_KL_mean": -722.4276733398438, + "KL/mean": -880.337158203125, + "KL/rejected_KL_mean": -1038.2467041015625, + "KL/std": 528.0697631835938, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0010385662317276, + "fcm_dpo/delta": 0.10328055918216705, + "fcm_dpo/margin": 315.8189697265625, + "fcm_dpo/q_t": 0.4342414438724518, + "grad_norm": 46.649322509765625, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.0891550779342651, + "logits/rejected": -1.091712474822998, + "logps/chosen": -786.5845947265625, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1123.3297119140625, + "loss": 1.186, + "margin_dpo/margin_mean": 315.8189697265625, + "margin_dpo/margin_std": 645.6528930664062, + "step": 623 + }, + { + "KL/chosen_KL_mean": -704.1187133789062, + "KL/mean": -950.7569580078125, + "KL/rejected_KL_mean": -1197.3951416015625, + "KL/std": 544.427490234375, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0009231247240677476, + "fcm_dpo/delta": -0.10394740104675293, + "fcm_dpo/margin": 493.2764892578125, + "fcm_dpo/q_t": 0.38469332456588745, + "grad_norm": 31.07668685913086, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.077886700630188, + "logits/rejected": -1.079150676727295, + "logps/chosen": -776.037353515625, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1294.5272216796875, + "loss": 1.0515, + "margin_dpo/margin_mean": 493.2764892578125, + "margin_dpo/margin_std": 586.5498046875, + "step": 624 + }, + { + "KL/chosen_KL_mean": -672.1764526367188, + "KL/mean": -935.4581298828125, + "KL/rejected_KL_mean": -1198.7398681640625, + "KL/std": 594.755615234375, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0008218766888603568, + "fcm_dpo/delta": -0.057466380298137665, + "fcm_dpo/margin": 526.5632934570312, + "fcm_dpo/q_t": 0.39506271481513977, + "grad_norm": 63.15570068359375, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.9775293469429016, + "logits/rejected": -1.0035572052001953, + "logps/chosen": -730.5185546875, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1284.8302001953125, + "loss": 1.0508, + "margin_dpo/margin_mean": 526.5633544921875, + "margin_dpo/margin_std": 566.92041015625, + "step": 625 + }, + { + "KL/chosen_KL_mean": -812.67529296875, + "KL/mean": -959.8338623046875, + "KL/rejected_KL_mean": -1106.9925537109375, + "KL/std": 628.1944580078125, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0009906254708766937, + "fcm_dpo/delta": 0.14670009911060333, + "fcm_dpo/margin": 294.3172607421875, + "fcm_dpo/q_t": 0.43976619839668274, + "grad_norm": 35.10493469238281, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.9490357637405396, + "logits/rejected": -0.9404639005661011, + "logps/chosen": -887.7879028320312, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1206.1812744140625, + "loss": 1.2132, + "margin_dpo/margin_mean": 294.3172607421875, + "margin_dpo/margin_std": 620.13720703125, + "step": 626 + }, + { + "KL/chosen_KL_mean": -542.8727416992188, + "KL/mean": -843.132080078125, + "KL/rejected_KL_mean": -1143.3914794921875, + "KL/std": 703.228271484375, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0008381834486499429, + "fcm_dpo/delta": -0.20297735929489136, + "fcm_dpo/margin": 600.518798828125, + "fcm_dpo/q_t": 0.3782247304916382, + "grad_norm": 29.68364715576172, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.9200704097747803, + "logits/rejected": -0.9946620464324951, + "logps/chosen": -590.6159057617188, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1250.14599609375, + "loss": 1.0535, + "margin_dpo/margin_mean": 600.518798828125, + "margin_dpo/margin_std": 814.1729736328125, + "step": 627 + }, + { + "KL/chosen_KL_mean": -741.293701171875, + "KL/mean": -966.7863159179688, + "KL/rejected_KL_mean": -1192.279052734375, + "KL/std": 596.0997924804688, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0007803476182743907, + "fcm_dpo/delta": 0.07325306534767151, + "fcm_dpo/margin": 450.9853515625, + "fcm_dpo/q_t": 0.423714816570282, + "grad_norm": 29.094398498535156, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.0479551553726196, + "logits/rejected": -1.0646214485168457, + "logps/chosen": -801.4765625, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1293.833740234375, + "loss": 1.1308, + "margin_dpo/margin_mean": 450.9853515625, + "margin_dpo/margin_std": 675.803466796875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -719.9290771484375, + "KL/mean": -929.900634765625, + "KL/rejected_KL_mean": -1139.872314453125, + "KL/std": 583.1702270507812, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0008786619873717427, + "fcm_dpo/delta": 0.04738582670688629, + "fcm_dpo/margin": 419.9432373046875, + "fcm_dpo/q_t": 0.4166840612888336, + "grad_norm": 32.80850601196289, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.0385822057724, + "logits/rejected": -1.0577049255371094, + "logps/chosen": -784.142578125, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1231.52587890625, + "loss": 1.112, + "margin_dpo/margin_mean": 419.9432373046875, + "margin_dpo/margin_std": 578.53662109375, + "step": 629 + }, + { + "KL/chosen_KL_mean": -647.0543212890625, + "KL/mean": -764.2266845703125, + "KL/rejected_KL_mean": -881.39892578125, + "KL/std": 556.714599609375, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0009413863299414515, + "fcm_dpo/delta": 0.056776098906993866, + "fcm_dpo/margin": 234.34454345703125, + "fcm_dpo/q_t": 0.4554150104522705, + "grad_norm": 49.36244201660156, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -0.9865130186080933, + "logits/rejected": -0.9569113254547119, + "logps/chosen": -706.3453369140625, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -964.9971923828125, + "loss": 1.2685, + "margin_dpo/margin_mean": 234.34454345703125, + "margin_dpo/margin_std": 668.56982421875, + "step": 630 + }, + { + "KL/chosen_KL_mean": -776.7286987304688, + "KL/mean": -965.1847534179688, + "KL/rejected_KL_mean": -1153.6407470703125, + "KL/std": 581.8084106445312, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0010326361516490579, + "fcm_dpo/delta": 0.016902513802051544, + "fcm_dpo/margin": 376.9120788574219, + "fcm_dpo/q_t": 0.4136263132095337, + "grad_norm": 44.76034164428711, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.0995469093322754, + "logits/rejected": -1.1115614175796509, + "logps/chosen": -836.1823120117188, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1234.59228515625, + "loss": 1.1518, + "margin_dpo/margin_mean": 376.9120788574219, + "margin_dpo/margin_std": 690.0167236328125, + "step": 631 + }, + { + "KL/chosen_KL_mean": -697.9989013671875, + "KL/mean": -890.870361328125, + "KL/rejected_KL_mean": -1083.741943359375, + "KL/std": 523.5582275390625, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.000998594332486391, + "fcm_dpo/delta": 0.01689380779862404, + "fcm_dpo/margin": 385.7430114746094, + "fcm_dpo/q_t": 0.41343018412590027, + "grad_norm": 48.563560485839844, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.047680139541626, + "logits/rejected": -1.0479519367218018, + "logps/chosen": -759.3504638671875, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1169.902099609375, + "loss": 1.1447, + "margin_dpo/margin_mean": 385.74298095703125, + "margin_dpo/margin_std": 662.6145629882812, + "step": 632 + }, + { + "KL/chosen_KL_mean": -745.3228149414062, + "KL/mean": -884.4315795898438, + "KL/rejected_KL_mean": -1023.540283203125, + "KL/std": 512.197509765625, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0012451799120754004, + "fcm_dpo/delta": 0.06800185889005661, + "fcm_dpo/margin": 278.2175598144531, + "fcm_dpo/q_t": 0.4261208176612854, + "grad_norm": 68.927978515625, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.06679105758667, + "logits/rejected": -1.0765254497528076, + "logps/chosen": -802.6009521484375, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1115.124267578125, + "loss": 1.2008, + "margin_dpo/margin_mean": 278.2175598144531, + "margin_dpo/margin_std": 601.500732421875, + "step": 633 + }, + { + "KL/chosen_KL_mean": -832.7714233398438, + "KL/mean": -965.0292358398438, + "KL/rejected_KL_mean": -1097.2869873046875, + "KL/std": 614.1533203125, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0013293168740347028, + "fcm_dpo/delta": 0.07503058761358261, + "fcm_dpo/margin": 264.5155334472656, + "fcm_dpo/q_t": 0.4308067560195923, + "grad_norm": 51.24019241333008, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.9571331739425659, + "logits/rejected": -0.9513822197914124, + "logps/chosen": -899.390380859375, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1204.41259765625, + "loss": 1.2102, + "margin_dpo/margin_mean": 264.5155334472656, + "margin_dpo/margin_std": 607.553466796875, + "step": 634 + }, + { + "KL/chosen_KL_mean": -726.7911376953125, + "KL/mean": -909.9793090820312, + "KL/rejected_KL_mean": -1093.16748046875, + "KL/std": 651.4637451171875, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.0012207648251205683, + "fcm_dpo/delta": -0.08999871462583542, + "fcm_dpo/margin": 366.37628173828125, + "fcm_dpo/q_t": 0.3993282914161682, + "grad_norm": 50.3961067199707, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.0328423976898193, + "logits/rejected": -1.0168393850326538, + "logps/chosen": -800.7467041015625, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1175.66796875, + "loss": 1.1187, + "margin_dpo/margin_mean": 366.37628173828125, + "margin_dpo/margin_std": 644.3758544921875, + "step": 635 + }, + { + "KL/chosen_KL_mean": -682.0125732421875, + "KL/mean": -899.14404296875, + "KL/rejected_KL_mean": -1116.275634765625, + "KL/std": 621.2393798828125, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0010760778095573187, + "fcm_dpo/delta": -0.12986694276332855, + "fcm_dpo/margin": 434.2630615234375, + "fcm_dpo/q_t": 0.38959363102912903, + "grad_norm": 29.961071014404297, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.9651399254798889, + "logits/rejected": -0.9689005613327026, + "logps/chosen": -741.6414794921875, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1198.25439453125, + "loss": 1.0948, + "margin_dpo/margin_mean": 434.2630310058594, + "margin_dpo/margin_std": 650.568603515625, + "step": 636 + }, + { + "KL/chosen_KL_mean": -688.02197265625, + "KL/mean": -933.2636108398438, + "KL/rejected_KL_mean": -1178.505126953125, + "KL/std": 643.35302734375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0008327056420966983, + "fcm_dpo/delta": -0.024866830557584763, + "fcm_dpo/margin": 490.4832458496094, + "fcm_dpo/q_t": 0.4043170213699341, + "grad_norm": 34.2825813293457, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.08000648021698, + "logits/rejected": -1.127310872077942, + "logps/chosen": -737.6746826171875, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1276.910400390625, + "loss": 1.0877, + "margin_dpo/margin_mean": 490.48321533203125, + "margin_dpo/margin_std": 645.675537109375, + "step": 637 + }, + { + "KL/chosen_KL_mean": -676.4507446289062, + "KL/mean": -868.2711791992188, + "KL/rejected_KL_mean": -1060.0916748046875, + "KL/std": 593.851318359375, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0008206696948036551, + "fcm_dpo/delta": -0.058288540691137314, + "fcm_dpo/margin": 383.64093017578125, + "fcm_dpo/q_t": 0.419683039188385, + "grad_norm": 30.65635871887207, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.9378705024719238, + "logits/rejected": -0.9226667881011963, + "logps/chosen": -734.607421875, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1139.39306640625, + "loss": 1.1863, + "margin_dpo/margin_mean": 383.64093017578125, + "margin_dpo/margin_std": 691.0709228515625, + "step": 638 + }, + { + "KL/chosen_KL_mean": -905.7219848632812, + "KL/mean": -986.7338256835938, + "KL/rejected_KL_mean": -1067.7457275390625, + "KL/std": 534.1470336914062, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0007728881319053471, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 162.023681640625, + "fcm_dpo/q_t": 0.47046107053756714, + "grad_norm": 76.31160736083984, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.9101752042770386, + "logits/rejected": -0.8493287563323975, + "logps/chosen": -978.045166015625, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1142.0206298828125, + "loss": 1.3279, + "margin_dpo/margin_mean": 162.023681640625, + "margin_dpo/margin_std": 657.9573364257812, + "step": 639 + }, + { + "KL/chosen_KL_mean": -690.0206298828125, + "KL/mean": -973.13671875, + "KL/rejected_KL_mean": -1256.2529296875, + "KL/std": 648.8939819335938, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0007562098908238113, + "fcm_dpo/delta": -0.05274926871061325, + "fcm_dpo/margin": 566.2322387695312, + "fcm_dpo/q_t": 0.3993333876132965, + "grad_norm": 46.08436965942383, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.9405654668807983, + "logits/rejected": -0.9669902920722961, + "logps/chosen": -746.1549682617188, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1364.85302734375, + "loss": 1.0815, + "margin_dpo/margin_mean": 566.2322387695312, + "margin_dpo/margin_std": 748.8364868164062, + "step": 640 + }, + { + "KL/chosen_KL_mean": -803.405517578125, + "KL/mean": -974.4388427734375, + "KL/rejected_KL_mean": -1145.47216796875, + "KL/std": 547.9729614257812, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0008681797189638019, + "fcm_dpo/delta": 0.14330008625984192, + "fcm_dpo/margin": 342.0666809082031, + "fcm_dpo/q_t": 0.4413103461265564, + "grad_norm": 27.740734100341797, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.0207037925720215, + "logits/rejected": -1.02461838722229, + "logps/chosen": -868.40234375, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1232.4644775390625, + "loss": 1.2043, + "margin_dpo/margin_mean": 342.06671142578125, + "margin_dpo/margin_std": 712.2955932617188, + "step": 641 + }, + { + "KL/chosen_KL_mean": -779.18896484375, + "KL/mean": -998.7464599609375, + "KL/rejected_KL_mean": -1218.3040771484375, + "KL/std": 729.4473266601562, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0009394697844982147, + "fcm_dpo/delta": -0.026245135813951492, + "fcm_dpo/margin": 439.1151123046875, + "fcm_dpo/q_t": 0.41486814618110657, + "grad_norm": 41.30564880371094, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.9439425468444824, + "logits/rejected": -0.9753347635269165, + "logps/chosen": -844.878173828125, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1328.546142578125, + "loss": 1.1663, + "margin_dpo/margin_mean": 439.1151428222656, + "margin_dpo/margin_std": 900.11669921875, + "step": 642 + }, + { + "KL/chosen_KL_mean": -674.6640625, + "KL/mean": -851.1025390625, + "KL/rejected_KL_mean": -1027.5411376953125, + "KL/std": 530.1317138671875, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0008457102812826633, + "fcm_dpo/delta": -0.019284440204501152, + "fcm_dpo/margin": 352.87701416015625, + "fcm_dpo/q_t": 0.4285130500793457, + "grad_norm": 34.299949645996094, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.0501961708068848, + "logits/rejected": -1.0501909255981445, + "logps/chosen": -726.614013671875, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1115.009521484375, + "loss": 1.1622, + "margin_dpo/margin_mean": 352.87701416015625, + "margin_dpo/margin_std": 535.4759521484375, + "step": 643 + }, + { + "KL/chosen_KL_mean": -669.6585693359375, + "KL/mean": -843.1557006835938, + "KL/rejected_KL_mean": -1016.65283203125, + "KL/std": 622.4733276367188, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0009780020918697119, + "fcm_dpo/delta": 0.08886226266622543, + "fcm_dpo/margin": 346.9942932128906, + "fcm_dpo/q_t": 0.43022337555885315, + "grad_norm": 48.09732437133789, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -0.9770164489746094, + "logits/rejected": -0.9840372800827026, + "logps/chosen": -728.67626953125, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1103.78955078125, + "loss": 1.1865, + "margin_dpo/margin_mean": 346.9942626953125, + "margin_dpo/margin_std": 714.44580078125, + "step": 644 + }, + { + "KL/chosen_KL_mean": -693.9857788085938, + "KL/mean": -786.16162109375, + "KL/rejected_KL_mean": -878.3374633789062, + "KL/std": 501.67999267578125, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0011059939861297607, + "fcm_dpo/delta": 0.04529920592904091, + "fcm_dpo/margin": 184.3516387939453, + "fcm_dpo/q_t": 0.45926517248153687, + "grad_norm": 131.60787963867188, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.989364743232727, + "logits/rejected": -0.97637939453125, + "logps/chosen": -749.86181640625, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -976.1182861328125, + "loss": 1.3317, + "margin_dpo/margin_mean": 184.3516387939453, + "margin_dpo/margin_std": 676.8275146484375, + "step": 645 + }, + { + "KL/chosen_KL_mean": -658.73046875, + "KL/mean": -810.00341796875, + "KL/rejected_KL_mean": -961.2762451171875, + "KL/std": 463.8369140625, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0011796164326369762, + "fcm_dpo/delta": 0.06660034507513046, + "fcm_dpo/margin": 302.5458068847656, + "fcm_dpo/q_t": 0.42494145035743713, + "grad_norm": 58.692771911621094, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -0.9866272211074829, + "logits/rejected": -0.9638226628303528, + "logps/chosen": -720.0062255859375, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1038.7821044921875, + "loss": 1.1639, + "margin_dpo/margin_mean": 302.54583740234375, + "margin_dpo/margin_std": 556.3814697265625, + "step": 646 + }, + { + "KL/chosen_KL_mean": -595.4979248046875, + "KL/mean": -783.5799560546875, + "KL/rejected_KL_mean": -971.6620483398438, + "KL/std": 541.9985961914062, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.001148381270468235, + "fcm_dpo/delta": -0.05802955478429794, + "fcm_dpo/margin": 376.1640319824219, + "fcm_dpo/q_t": 0.3967111110687256, + "grad_norm": 51.125606536865234, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.9212703704833984, + "logits/rejected": -0.9506291151046753, + "logps/chosen": -650.350341796875, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1065.181396484375, + "loss": 1.0894, + "margin_dpo/margin_mean": 376.1640319824219, + "margin_dpo/margin_std": 538.5177001953125, + "step": 647 + }, + { + "KL/chosen_KL_mean": -622.4876708984375, + "KL/mean": -907.7584228515625, + "KL/rejected_KL_mean": -1193.0291748046875, + "KL/std": 659.01708984375, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0008362603839486837, + "fcm_dpo/delta": -0.17148783802986145, + "fcm_dpo/margin": 570.54150390625, + "fcm_dpo/q_t": 0.3812508285045624, + "grad_norm": 28.91020965576172, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.062262773513794, + "logits/rejected": -1.119946002960205, + "logps/chosen": -676.6591186523438, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1291.741943359375, + "loss": 1.0583, + "margin_dpo/margin_mean": 570.5415649414062, + "margin_dpo/margin_std": 795.2802734375, + "step": 648 + }, + { + "KL/chosen_KL_mean": -647.7181396484375, + "KL/mean": -777.0692138671875, + "KL/rejected_KL_mean": -906.4202880859375, + "KL/std": 489.11322021484375, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0008656082791276276, + "fcm_dpo/delta": 0.07774749398231506, + "fcm_dpo/margin": 258.7022399902344, + "fcm_dpo/q_t": 0.4531601071357727, + "grad_norm": 29.970624923706055, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.0473252534866333, + "logits/rejected": -1.0361632108688354, + "logps/chosen": -710.198486328125, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -986.4974975585938, + "loss": 1.2466, + "margin_dpo/margin_mean": 258.7022399902344, + "margin_dpo/margin_std": 601.4564208984375, + "step": 649 + }, + { + "KL/chosen_KL_mean": -711.3486328125, + "KL/mean": -921.720703125, + "KL/rejected_KL_mean": -1132.0926513671875, + "KL/std": 623.22998046875, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0009292041650041938, + "fcm_dpo/delta": 0.013731414452195168, + "fcm_dpo/margin": 420.7440185546875, + "fcm_dpo/q_t": 0.41418665647506714, + "grad_norm": 31.185792922973633, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -0.9931234121322632, + "logits/rejected": -1.013329029083252, + "logps/chosen": -767.4414672851562, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1230.357421875, + "loss": 1.1316, + "margin_dpo/margin_mean": 420.7440185546875, + "margin_dpo/margin_std": 705.7841796875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -472.3923034667969, + "KL/mean": -752.0784912109375, + "KL/rejected_KL_mean": -1031.7647705078125, + "KL/std": 590.5972900390625, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0008050062460824847, + "fcm_dpo/delta": -0.09981894493103027, + "fcm_dpo/margin": 559.3723754882812, + "fcm_dpo/q_t": 0.3874790072441101, + "grad_norm": 33.94593048095703, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.0268137454986572, + "logits/rejected": -1.0827641487121582, + "logps/chosen": -515.8177490234375, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1131.72265625, + "loss": 1.0412, + "margin_dpo/margin_mean": 559.3724365234375, + "margin_dpo/margin_std": 614.9459228515625, + "step": 651 + }, + { + "KL/chosen_KL_mean": -589.6246337890625, + "KL/mean": -800.077392578125, + "KL/rejected_KL_mean": -1010.5301513671875, + "KL/std": 563.1753540039062, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0008659152081236243, + "fcm_dpo/delta": 0.053139444440603256, + "fcm_dpo/margin": 420.905517578125, + "fcm_dpo/q_t": 0.42015212774276733, + "grad_norm": 43.597076416015625, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.026230812072754, + "logits/rejected": -1.0473260879516602, + "logps/chosen": -652.201416015625, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1122.2979736328125, + "loss": 1.1242, + "margin_dpo/margin_mean": 420.905517578125, + "margin_dpo/margin_std": 636.0498046875, + "step": 652 + }, + { + "KL/chosen_KL_mean": -745.919677734375, + "KL/mean": -946.8770751953125, + "KL/rejected_KL_mean": -1147.834228515625, + "KL/std": 630.18896484375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0009330883622169495, + "fcm_dpo/delta": 0.03911864012479782, + "fcm_dpo/margin": 401.91461181640625, + "fcm_dpo/q_t": 0.4190702438354492, + "grad_norm": 33.804931640625, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.0814123153686523, + "logits/rejected": -1.107730507850647, + "logps/chosen": -807.03271484375, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1251.083984375, + "loss": 1.1333, + "margin_dpo/margin_mean": 401.9146423339844, + "margin_dpo/margin_std": 663.3088989257812, + "step": 653 + }, + { + "KL/chosen_KL_mean": -651.044677734375, + "KL/mean": -828.8569946289062, + "KL/rejected_KL_mean": -1006.66943359375, + "KL/std": 504.73419189453125, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0010234671644866467, + "fcm_dpo/delta": 0.05655752867460251, + "fcm_dpo/margin": 355.62469482421875, + "fcm_dpo/q_t": 0.42311620712280273, + "grad_norm": 37.25580978393555, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.0293757915496826, + "logits/rejected": -1.0464175939559937, + "logps/chosen": -712.7728271484375, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1105.4432373046875, + "loss": 1.141, + "margin_dpo/margin_mean": 355.62469482421875, + "margin_dpo/margin_std": 592.0263671875, + "step": 654 + }, + { + "KL/chosen_KL_mean": -607.3045654296875, + "KL/mean": -795.6822509765625, + "KL/rejected_KL_mean": -984.0599365234375, + "KL/std": 513.406005859375, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0010560275986790657, + "fcm_dpo/delta": 0.003467606846243143, + "fcm_dpo/margin": 376.75543212890625, + "fcm_dpo/q_t": 0.4126419126987457, + "grad_norm": 35.243900299072266, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -0.9929590225219727, + "logits/rejected": -1.032914638519287, + "logps/chosen": -656.88134765625, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1082.351806640625, + "loss": 1.1245, + "margin_dpo/margin_mean": 376.75543212890625, + "margin_dpo/margin_std": 618.8502197265625, + "step": 655 + }, + { + "KL/chosen_KL_mean": -648.4987182617188, + "KL/mean": -975.326904296875, + "KL/rejected_KL_mean": -1302.1551513671875, + "KL/std": 691.7890014648438, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0007785690249875188, + "fcm_dpo/delta": -0.22655422985553741, + "fcm_dpo/margin": 653.6564331054688, + "fcm_dpo/q_t": 0.36488282680511475, + "grad_norm": 64.71890258789062, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.899747371673584, + "logits/rejected": -0.968382716178894, + "logps/chosen": -701.0481567382812, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1415.829833984375, + "loss": 1.013, + "margin_dpo/margin_mean": 653.6564331054688, + "margin_dpo/margin_std": 684.8790283203125, + "step": 656 + }, + { + "KL/chosen_KL_mean": -607.3179931640625, + "KL/mean": -871.7550659179688, + "KL/rejected_KL_mean": -1136.192138671875, + "KL/std": 643.2208862304688, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0007102171657606959, + "fcm_dpo/delta": 0.039193179458379745, + "fcm_dpo/margin": 528.8741455078125, + "fcm_dpo/q_t": 0.415811151266098, + "grad_norm": 30.748796463012695, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.0481717586517334, + "logits/rejected": -1.0999813079833984, + "logps/chosen": -654.0185546875, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1234.1070556640625, + "loss": 1.0999, + "margin_dpo/margin_mean": 528.8741455078125, + "margin_dpo/margin_std": 680.64990234375, + "step": 657 + }, + { + "KL/chosen_KL_mean": -668.5402221679688, + "KL/mean": -876.7470703125, + "KL/rejected_KL_mean": -1084.9539794921875, + "KL/std": 520.5706176757812, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0008155145333148539, + "fcm_dpo/delta": 0.0921507477760315, + "fcm_dpo/margin": 416.4137268066406, + "fcm_dpo/q_t": 0.4271154999732971, + "grad_norm": 30.934783935546875, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -0.9990421533584595, + "logits/rejected": -1.0265355110168457, + "logps/chosen": -729.4984130859375, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1180.8934326171875, + "loss": 1.1332, + "margin_dpo/margin_mean": 416.4136962890625, + "margin_dpo/margin_std": 572.531494140625, + "step": 658 + }, + { + "KL/chosen_KL_mean": -589.103515625, + "KL/mean": -787.4786376953125, + "KL/rejected_KL_mean": -985.8536376953125, + "KL/std": 501.25604248046875, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0009247631533071399, + "fcm_dpo/delta": 0.0519876554608345, + "fcm_dpo/margin": 396.7500915527344, + "fcm_dpo/q_t": 0.42018935084342957, + "grad_norm": 30.552549362182617, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.9165958762168884, + "logits/rejected": -0.885309100151062, + "logps/chosen": -665.8465576171875, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1073.3245849609375, + "loss": 1.1172, + "margin_dpo/margin_mean": 396.7500915527344, + "margin_dpo/margin_std": 562.6476440429688, + "step": 659 + }, + { + "KL/chosen_KL_mean": -651.0494384765625, + "KL/mean": -908.062744140625, + "KL/rejected_KL_mean": -1165.076171875, + "KL/std": 607.6834716796875, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0008679507300257683, + "fcm_dpo/delta": -0.08382175117731094, + "fcm_dpo/margin": 514.0266723632812, + "fcm_dpo/q_t": 0.39148008823394775, + "grad_norm": 46.10919952392578, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.017472505569458, + "logits/rejected": -1.033474326133728, + "logps/chosen": -710.0972900390625, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1241.0361328125, + "loss": 1.0632, + "margin_dpo/margin_mean": 514.0266723632812, + "margin_dpo/margin_std": 650.8546142578125, + "step": 660 + }, + { + "KL/chosen_KL_mean": -573.46484375, + "KL/mean": -808.244140625, + "KL/rejected_KL_mean": -1043.0234375, + "KL/std": 656.3875732421875, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0008553254883736372, + "fcm_dpo/delta": -0.0076020583510398865, + "fcm_dpo/margin": 469.55877685546875, + "fcm_dpo/q_t": 0.408011257648468, + "grad_norm": 39.97681427001953, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.0107231140136719, + "logits/rejected": -1.019051194190979, + "logps/chosen": -624.1387939453125, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1129.0291748046875, + "loss": 1.0917, + "margin_dpo/margin_mean": 469.5587463378906, + "margin_dpo/margin_std": 631.5257568359375, + "step": 661 + }, + { + "KL/chosen_KL_mean": -674.645263671875, + "KL/mean": -845.860595703125, + "KL/rejected_KL_mean": -1017.075927734375, + "KL/std": 547.025146484375, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0009593941504135728, + "fcm_dpo/delta": 0.10353372991085052, + "fcm_dpo/margin": 342.43072509765625, + "fcm_dpo/q_t": 0.43190035223960876, + "grad_norm": 30.81041717529297, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.9726383090019226, + "logits/rejected": -0.969027042388916, + "logps/chosen": -743.9063110351562, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1106.1318359375, + "loss": 1.1861, + "margin_dpo/margin_mean": 342.43072509765625, + "margin_dpo/margin_std": 687.1112060546875, + "step": 662 + }, + { + "KL/chosen_KL_mean": -622.2049560546875, + "KL/mean": -859.8319091796875, + "KL/rejected_KL_mean": -1097.458740234375, + "KL/std": 645.55224609375, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.000906460452824831, + "fcm_dpo/delta": -0.05661872774362564, + "fcm_dpo/margin": 475.2537536621094, + "fcm_dpo/q_t": 0.40141403675079346, + "grad_norm": 25.596982955932617, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.029599666595459, + "logits/rejected": -1.055091381072998, + "logps/chosen": -687.0839233398438, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1211.3841552734375, + "loss": 1.1023, + "margin_dpo/margin_mean": 475.25372314453125, + "margin_dpo/margin_std": 764.5504760742188, + "step": 663 + }, + { + "KL/chosen_KL_mean": -656.9991455078125, + "KL/mean": -909.43798828125, + "KL/rejected_KL_mean": -1161.876708984375, + "KL/std": 617.7022705078125, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0008173753740265965, + "fcm_dpo/delta": -0.0294804647564888, + "fcm_dpo/margin": 504.87762451171875, + "fcm_dpo/q_t": 0.40310847759246826, + "grad_norm": 22.38481330871582, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -0.987866997718811, + "logits/rejected": -1.0178110599517822, + "logps/chosen": -717.8876342773438, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1267.3984375, + "loss": 1.0872, + "margin_dpo/margin_mean": 504.87762451171875, + "margin_dpo/margin_std": 675.4814453125, + "step": 664 + }, + { + "KL/chosen_KL_mean": -591.2396240234375, + "KL/mean": -797.2940673828125, + "KL/rejected_KL_mean": -1003.3485717773438, + "KL/std": 507.349609375, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0009045639308169484, + "fcm_dpo/delta": 0.04246987774968147, + "fcm_dpo/margin": 412.10906982421875, + "fcm_dpo/q_t": 0.4166688919067383, + "grad_norm": 44.036556243896484, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.0652823448181152, + "logits/rejected": -1.0855448246002197, + "logps/chosen": -651.8037109375, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1088.157470703125, + "loss": 1.099, + "margin_dpo/margin_mean": 412.10906982421875, + "margin_dpo/margin_std": 510.36553955078125, + "step": 665 + }, + { + "KL/chosen_KL_mean": -635.3843994140625, + "KL/mean": -849.865234375, + "KL/rejected_KL_mean": -1064.3460693359375, + "KL/std": 543.2855224609375, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0009157652966678143, + "fcm_dpo/delta": 0.011229622177779675, + "fcm_dpo/margin": 428.96173095703125, + "fcm_dpo/q_t": 0.4111603796482086, + "grad_norm": 29.72758674621582, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.9271786212921143, + "logits/rejected": -0.9376795291900635, + "logps/chosen": -699.8043212890625, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1160.23779296875, + "loss": 1.1035, + "margin_dpo/margin_mean": 428.96173095703125, + "margin_dpo/margin_std": 589.12890625, + "step": 666 + }, + { + "KL/chosen_KL_mean": -728.5592041015625, + "KL/mean": -976.642822265625, + "KL/rejected_KL_mean": -1224.7265625, + "KL/std": 594.4561767578125, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0008521339623257518, + "fcm_dpo/delta": -0.04440337419509888, + "fcm_dpo/margin": 496.16729736328125, + "fcm_dpo/q_t": 0.39933979511260986, + "grad_norm": 36.45888137817383, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.9821093082427979, + "logits/rejected": -0.9815555810928345, + "logps/chosen": -797.8362426757812, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1312.56201171875, + "loss": 1.0793, + "margin_dpo/margin_mean": 496.1672668457031, + "margin_dpo/margin_std": 653.2168579101562, + "step": 667 + }, + { + "KL/chosen_KL_mean": -755.283447265625, + "KL/mean": -929.7744140625, + "KL/rejected_KL_mean": -1104.265380859375, + "KL/std": 615.79931640625, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0010012466227635741, + "fcm_dpo/delta": 0.061918459832668304, + "fcm_dpo/margin": 348.9818420410156, + "fcm_dpo/q_t": 0.4277912974357605, + "grad_norm": 45.25949478149414, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.0155105590820312, + "logits/rejected": -1.0245213508605957, + "logps/chosen": -827.887451171875, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1208.00439453125, + "loss": 1.1889, + "margin_dpo/margin_mean": 348.9818420410156, + "margin_dpo/margin_std": 726.23681640625, + "step": 668 + }, + { + "KL/chosen_KL_mean": -597.3612060546875, + "KL/mean": -849.0230102539062, + "KL/rejected_KL_mean": -1100.6846923828125, + "KL/std": 599.0240478515625, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0008747231913730502, + "fcm_dpo/delta": -0.07254499942064285, + "fcm_dpo/margin": 503.323486328125, + "fcm_dpo/q_t": 0.393608957529068, + "grad_norm": 22.192842483520508, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -0.9759036302566528, + "logits/rejected": -0.9822956323623657, + "logps/chosen": -643.4776611328125, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1178.609130859375, + "loss": 1.0742, + "margin_dpo/margin_mean": 503.323486328125, + "margin_dpo/margin_std": 677.1585693359375, + "step": 669 + }, + { + "KL/chosen_KL_mean": -544.9779052734375, + "KL/mean": -801.1505737304688, + "KL/rejected_KL_mean": -1057.3232421875, + "KL/std": 542.9509887695312, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0008237832807935774, + "fcm_dpo/delta": -0.039803534746170044, + "fcm_dpo/margin": 512.3453369140625, + "fcm_dpo/q_t": 0.3997776508331299, + "grad_norm": 40.426666259765625, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.9458330273628235, + "logits/rejected": -0.9657979011535645, + "logps/chosen": -607.3236694335938, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1154.2637939453125, + "loss": 1.0816, + "margin_dpo/margin_mean": 512.3453369140625, + "margin_dpo/margin_std": 667.56640625, + "step": 670 + }, + { + "KL/chosen_KL_mean": -693.818603515625, + "KL/mean": -904.692626953125, + "KL/rejected_KL_mean": -1115.5667724609375, + "KL/std": 550.0133056640625, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.0008690550457686186, + "fcm_dpo/delta": 0.04318936541676521, + "fcm_dpo/margin": 421.74822998046875, + "fcm_dpo/q_t": 0.41694390773773193, + "grad_norm": 26.835704803466797, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.0066074132919312, + "logits/rejected": -1.0331590175628662, + "logps/chosen": -741.8187255859375, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1199.3861083984375, + "loss": 1.1341, + "margin_dpo/margin_mean": 421.74822998046875, + "margin_dpo/margin_std": 664.017333984375, + "step": 671 + }, + { + "KL/chosen_KL_mean": -766.8887329101562, + "KL/mean": -968.2491455078125, + "KL/rejected_KL_mean": -1169.6094970703125, + "KL/std": 652.8719482421875, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0009030759101733565, + "fcm_dpo/delta": 0.057317838072776794, + "fcm_dpo/margin": 402.7208251953125, + "fcm_dpo/q_t": 0.4237225651741028, + "grad_norm": 62.17927932739258, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.085627794265747, + "logits/rejected": -1.0907000303268433, + "logps/chosen": -825.4720458984375, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1262.749755859375, + "loss": 1.1636, + "margin_dpo/margin_mean": 402.7208251953125, + "margin_dpo/margin_std": 737.9989013671875, + "step": 672 + }, + { + "KL/chosen_KL_mean": -674.4365234375, + "KL/mean": -871.0324096679688, + "KL/rejected_KL_mean": -1067.62841796875, + "KL/std": 548.711181640625, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0009767746087163687, + "fcm_dpo/delta": 0.02593318186700344, + "fcm_dpo/margin": 393.1917724609375, + "fcm_dpo/q_t": 0.41595256328582764, + "grad_norm": 40.66542053222656, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.0082206726074219, + "logits/rejected": -1.0116488933563232, + "logps/chosen": -721.15966796875, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1152.924560546875, + "loss": 1.119, + "margin_dpo/margin_mean": 393.1917724609375, + "margin_dpo/margin_std": 624.88330078125, + "step": 673 + }, + { + "KL/chosen_KL_mean": -571.8458862304688, + "KL/mean": -831.9334716796875, + "KL/rejected_KL_mean": -1092.0211181640625, + "KL/std": 578.303466796875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0008713127463124692, + "fcm_dpo/delta": -0.09795575588941574, + "fcm_dpo/margin": 520.1752319335938, + "fcm_dpo/q_t": 0.39291104674339294, + "grad_norm": 35.70085144042969, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.9462261199951172, + "logits/rejected": -0.9662339091300964, + "logps/chosen": -617.2913818359375, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1162.067138671875, + "loss": 1.0669, + "margin_dpo/margin_mean": 520.1752319335938, + "margin_dpo/margin_std": 695.502685546875, + "step": 674 + }, + { + "KL/chosen_KL_mean": -642.9573974609375, + "KL/mean": -909.2418212890625, + "KL/rejected_KL_mean": -1175.526123046875, + "KL/std": 614.2586059570312, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0007445673691108823, + "fcm_dpo/delta": -0.013315960764884949, + "fcm_dpo/margin": 532.56884765625, + "fcm_dpo/q_t": 0.4076732397079468, + "grad_norm": 23.048574447631836, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.0072154998779297, + "logits/rejected": -1.0248122215270996, + "logps/chosen": -687.1336669921875, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1249.6181640625, + "loss": 1.0911, + "margin_dpo/margin_mean": 532.56884765625, + "margin_dpo/margin_std": 652.190185546875, + "step": 675 + }, + { + "KL/chosen_KL_mean": -695.682373046875, + "KL/mean": -932.237060546875, + "KL/rejected_KL_mean": -1168.7918701171875, + "KL/std": 565.1691284179688, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0008274284191429615, + "fcm_dpo/delta": 0.013139697723090649, + "fcm_dpo/margin": 473.10943603515625, + "fcm_dpo/q_t": 0.409980833530426, + "grad_norm": 23.929630279541016, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.9380951523780823, + "logits/rejected": -0.9486560821533203, + "logps/chosen": -767.0808715820312, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1257.150634765625, + "loss": 1.086, + "margin_dpo/margin_mean": 473.1094665527344, + "margin_dpo/margin_std": 570.78515625, + "step": 676 + }, + { + "KL/chosen_KL_mean": -701.174560546875, + "KL/mean": -933.6365356445312, + "KL/rejected_KL_mean": -1166.0985107421875, + "KL/std": 588.0880126953125, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0008385643595829606, + "fcm_dpo/delta": 0.01662612520158291, + "fcm_dpo/margin": 464.9239501953125, + "fcm_dpo/q_t": 0.4153476655483246, + "grad_norm": 24.641782760620117, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.0368056297302246, + "logits/rejected": -1.0287206172943115, + "logps/chosen": -757.7020263671875, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1244.324951171875, + "loss": 1.1094, + "margin_dpo/margin_mean": 464.9239196777344, + "margin_dpo/margin_std": 704.014892578125, + "step": 677 + }, + { + "KL/chosen_KL_mean": -589.749267578125, + "KL/mean": -871.703369140625, + "KL/rejected_KL_mean": -1153.657470703125, + "KL/std": 666.5526123046875, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0007895482704043388, + "fcm_dpo/delta": -0.08192657679319382, + "fcm_dpo/margin": 563.908203125, + "fcm_dpo/q_t": 0.39262598752975464, + "grad_norm": 28.306751251220703, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.9746444821357727, + "logits/rejected": -1.0132948160171509, + "logps/chosen": -635.8837280273438, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1234.2620849609375, + "loss": 1.0705, + "margin_dpo/margin_mean": 563.9081420898438, + "margin_dpo/margin_std": 737.7059326171875, + "step": 678 + }, + { + "KL/chosen_KL_mean": -681.0501708984375, + "KL/mean": -873.888916015625, + "KL/rejected_KL_mean": -1066.727783203125, + "KL/std": 521.9443359375, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0008484461577609181, + "fcm_dpo/delta": 0.10628005862236023, + "fcm_dpo/margin": 385.67755126953125, + "fcm_dpo/q_t": 0.43186846375465393, + "grad_norm": 29.822158813476562, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -0.9917019605636597, + "logits/rejected": -0.9838023781776428, + "logps/chosen": -731.3450927734375, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1143.3258056640625, + "loss": 1.1502, + "margin_dpo/margin_mean": 385.67755126953125, + "margin_dpo/margin_std": 617.3707275390625, + "step": 679 + }, + { + "KL/chosen_KL_mean": -673.279541015625, + "KL/mean": -923.5628051757812, + "KL/rejected_KL_mean": -1173.8460693359375, + "KL/std": 654.016845703125, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0008149506174959242, + "fcm_dpo/delta": -0.02367626130580902, + "fcm_dpo/margin": 500.56646728515625, + "fcm_dpo/q_t": 0.4036046266555786, + "grad_norm": 44.08322525024414, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -0.9790595173835754, + "logits/rejected": -1.0117442607879639, + "logps/chosen": -750.1953125, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1286.2308349609375, + "loss": 1.1057, + "margin_dpo/margin_mean": 500.56646728515625, + "margin_dpo/margin_std": 737.4908447265625, + "step": 680 + }, + { + "KL/chosen_KL_mean": -678.2127685546875, + "KL/mean": -878.902587890625, + "KL/rejected_KL_mean": -1079.592529296875, + "KL/std": 559.0294799804688, + "epoch": 1.0, + "fcm_dpo/beta": 0.0008184043690562248, + "fcm_dpo/delta": -0.0384669154882431, + "fcm_dpo/margin": 401.3796691894531, + "fcm_dpo/q_t": 0.4222037196159363, + "grad_norm": 26.581893920898438, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.0419981479644775, + "logits/rejected": -1.0603480339050293, + "logps/chosen": -739.1700439453125, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1168.150390625, + "loss": 1.1543, + "margin_dpo/margin_mean": 401.379638671875, + "margin_dpo/margin_std": 625.6992797851562, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.1280472663705745, + "train_runtime": 1739.8346, + "train_samples_per_second": 25.059, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}