commit 4a3ecef1eeea115e69be379c463c20be4108fdfb Author: ModelHub XC Date: Wed May 13 20:03:24 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.43 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb1380e --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.43 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-s_star-0.4-eta-0.1-q_t-0.43 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..1416301 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1157032691609667, + "train_runtime": 1740.0089, + "train_samples": 43598, + "train_samples_per_second": 25.056, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..f2d035b --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12df28920ef5d5aa383af73f79ee8889c1338b7abe75a08127b96cba442a96d9 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..5e995df --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04dd13a22754547f20a1403e3cec453756e9dd57a011c2440dffe6e275c0bae +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..9e2c95d --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603b50aa7b28975a797b1935d363749752a7bed5e9dd2c7c10dca08a096d770a +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..0822d48 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a332d11e610ec12fe086823ac0c82ed47e3bb8b69ad6f784d3d3acefc327e4f3 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..4303ad4 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f16fcc7721c7c65868ae4f5ebb314fe09794d84029cf6028b0f3341cadc6b80 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..a5ecdd5 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2081bc09727797a33754b381c3ac6dc1b4e2a6dcfdeffb2537d6852982bdbbfe +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..dbdfaa1 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:594512594af2b4dbf1b435bc2902f200b2ea6cf3d69ceacc91e97af2a91dce76 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..1416301 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.1157032691609667, + "train_runtime": 1740.0089, + "train_samples": 43598, + "train_samples_per_second": 25.056, + "train_steps_per_second": 0.391 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..375a26d --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.51420593261719, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.21273040771484, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.014606475830078125, + "KL/mean": 0.019288957118988037, + "KL/rejected_KL_mean": 0.023973464965820312, + "KL/std": 0.24553033709526062, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.00936734676361084, + "fcm_dpo/q_t": 0.5002347230911255, + "grad_norm": 70.9432373046875, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4816523790359497, + "logits/rejected": -0.44211775064468384, + "logps/chosen": -60.96698760986328, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.64862060546875, + "loss": 1.3876, + "margin_dpo/margin_mean": -0.009366899728775024, + "margin_dpo/margin_std": 0.365927517414093, + "step": 3 + }, + { + "KL/chosen_KL_mean": -0.009187698364257812, + "KL/mean": -0.006194323301315308, + "KL/rejected_KL_mean": -0.003200531005859375, + "KL/std": 0.22787970304489136, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.005986988544464111, + "fcm_dpo/q_t": 0.5001498460769653, + "grad_norm": 71.68559265136719, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.468938946723938, + "logits/rejected": -0.44135916233062744, + "logps/chosen": -56.77690124511719, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.65031433105469, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.005987375974655151, + "margin_dpo/margin_std": 0.33422666788101196, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.01798248291015625, + "KL/mean": 0.006481155753135681, + "KL/rejected_KL_mean": -0.005016326904296875, + "KL/std": 0.2760767936706543, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02300247550010681, + "fcm_dpo/q_t": 0.4994247555732727, + "grad_norm": 90.19357299804688, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.514616847038269, + "logits/rejected": -0.4708850681781769, + "logps/chosen": -53.841392517089844, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.15420532226562, + "loss": 1.3844, + "margin_dpo/margin_mean": 0.02300289273262024, + "margin_dpo/margin_std": 0.39623090624809265, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.034801483154296875, + "KL/mean": -0.01997770369052887, + "KL/rejected_KL_mean": -0.005153656005859375, + "KL/std": 0.2839451730251312, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02964690327644348, + "fcm_dpo/q_t": 0.500741720199585, + "grad_norm": 92.58969116210938, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5107317566871643, + "logits/rejected": -0.4688287377357483, + "logps/chosen": -63.04228591918945, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.65049743652344, + "loss": 1.3897, + "margin_dpo/margin_mean": -0.029647350311279297, + "margin_dpo/margin_std": 0.39619821310043335, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.03380584716796875, + "KL/mean": 0.025876715779304504, + "KL/rejected_KL_mean": 0.017948150634765625, + "KL/std": 0.32117217779159546, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.015859276056289673, + "fcm_dpo/q_t": 0.4996029734611511, + "grad_norm": 82.21163177490234, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5053023099899292, + "logits/rejected": -0.4714616537094116, + "logps/chosen": -57.74101257324219, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.90264892578125, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.015859544277191162, + "margin_dpo/margin_std": 0.4937620460987091, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.024888992309570312, + "KL/mean": 0.04606606066226959, + "KL/rejected_KL_mean": 0.06724929809570312, + "KL/std": 0.24709418416023254, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.042354851961135864, + "fcm_dpo/q_t": 0.5010585784912109, + "grad_norm": 79.50102233886719, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5138094425201416, + "logits/rejected": -0.48857229948043823, + "logps/chosen": -58.69114685058594, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.24417114257812, + "loss": 1.3908, + "margin_dpo/margin_mean": -0.04235494136810303, + "margin_dpo/margin_std": 0.31987226009368896, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.009906768798828125, + "KL/mean": 0.017921432852745056, + "KL/rejected_KL_mean": 0.025936126708984375, + "KL/std": 0.2927726209163666, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01602998375892639, + "fcm_dpo/q_t": 0.5004010200500488, + "grad_norm": 85.36547088623047, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.4845085144042969, + "logits/rejected": -0.4371638894081116, + "logps/chosen": -69.85693359375, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.57672882080078, + "loss": 1.3884, + "margin_dpo/margin_mean": -0.016030579805374146, + "margin_dpo/margin_std": 0.417758584022522, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.04011344909667969, + "KL/mean": -0.0022094547748565674, + "KL/rejected_KL_mean": -0.04453277587890625, + "KL/std": 0.27158689498901367, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08464503288269043, + "fcm_dpo/q_t": 0.4978843927383423, + "grad_norm": 70.49663543701172, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.4809609055519104, + "logits/rejected": -0.4371563196182251, + "logps/chosen": -48.31757354736328, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.41659545898438, + "loss": 1.3782, + "margin_dpo/margin_mean": 0.08464545011520386, + "margin_dpo/margin_std": 0.3670717477798462, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.024618148803710938, + "KL/mean": 0.017558962106704712, + "KL/rejected_KL_mean": 0.010501861572265625, + "KL/std": 0.2545170485973358, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01411294937133789, + "fcm_dpo/q_t": 0.4996476173400879, + "grad_norm": 68.84070587158203, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.46836793422698975, + "logits/rejected": -0.4438410997390747, + "logps/chosen": -52.99223709106445, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.7698745727539, + "loss": 1.3852, + "margin_dpo/margin_mean": 0.01411372423171997, + "margin_dpo/margin_std": 0.37134677171707153, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.027496337890625, + "KL/mean": -0.03281432390213013, + "KL/rejected_KL_mean": -0.03812408447265625, + "KL/std": 0.2752145528793335, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.010634899139404297, + "fcm_dpo/q_t": 0.49973398447036743, + "grad_norm": 89.73855590820312, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5604701042175293, + "logits/rejected": -0.5252009034156799, + "logps/chosen": -61.83293151855469, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.89639282226562, + "loss": 1.3856, + "margin_dpo/margin_mean": 0.010634094476699829, + "margin_dpo/margin_std": 0.3748665452003479, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.04120445251464844, + "KL/mean": 0.010931193828582764, + "KL/rejected_KL_mean": -0.019344329833984375, + "KL/std": 0.24759797751903534, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.060551345348358154, + "fcm_dpo/q_t": 0.49848711490631104, + "grad_norm": 79.46299743652344, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.45892220735549927, + "logits/rejected": -0.4291881322860718, + "logps/chosen": -64.21914672851562, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.22242736816406, + "loss": 1.3805, + "margin_dpo/margin_mean": 0.06055182218551636, + "margin_dpo/margin_std": 0.327784925699234, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.023923873901367188, + "KL/mean": -0.041747644543647766, + "KL/rejected_KL_mean": -0.059574127197265625, + "KL/std": 0.26473259925842285, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03565385937690735, + "fcm_dpo/q_t": 0.49910902976989746, + "grad_norm": 84.94186401367188, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.46589577198028564, + "logits/rejected": -0.4261009693145752, + "logps/chosen": -58.134132385253906, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.10665893554688, + "loss": 1.3832, + "margin_dpo/margin_mean": 0.03565409779548645, + "margin_dpo/margin_std": 0.43024545907974243, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.0184326171875, + "KL/mean": -0.04868185520172119, + "KL/rejected_KL_mean": -0.07892990112304688, + "KL/std": 0.26160433888435364, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.060504019260406494, + "fcm_dpo/q_t": 0.49848800897598267, + "grad_norm": 64.09009552001953, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.49907413125038147, + "logits/rejected": -0.48077064752578735, + "logps/chosen": -56.98534393310547, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.8875732421875, + "loss": 1.3807, + "margin_dpo/margin_mean": 0.06050407886505127, + "margin_dpo/margin_std": 0.39841049909591675, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.020772933959960938, + "KL/mean": -0.03767618536949158, + "KL/rejected_KL_mean": -0.05457305908203125, + "KL/std": 0.2748359739780426, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03379744291305542, + "fcm_dpo/q_t": 0.4991554021835327, + "grad_norm": 83.73448944091797, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5354863405227661, + "logits/rejected": -0.49548211693763733, + "logps/chosen": -61.76066589355469, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.42404174804688, + "loss": 1.3833, + "margin_dpo/margin_mean": 0.03379681706428528, + "margin_dpo/margin_std": 0.3752933144569397, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.03517913818359375, + "KL/mean": -0.02083699405193329, + "KL/rejected_KL_mean": -0.07685089111328125, + "KL/std": 0.27744123339653015, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.11203554272651672, + "fcm_dpo/q_t": 0.4972001910209656, + "grad_norm": 78.40747833251953, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.48720961809158325, + "logits/rejected": -0.4483189880847931, + "logps/chosen": -67.67515563964844, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.45550537109375, + "loss": 1.3754, + "margin_dpo/margin_mean": 0.11203566193580627, + "margin_dpo/margin_std": 0.332059383392334, + "step": 17 + }, + { + "KL/chosen_KL_mean": -0.012210845947265625, + "KL/mean": -0.038145169615745544, + "KL/rejected_KL_mean": -0.06407928466796875, + "KL/std": 0.2823082506656647, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05187109112739563, + "fcm_dpo/q_t": 0.49870359897613525, + "grad_norm": 81.85424041748047, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.4941648244857788, + "logits/rejected": -0.43878093361854553, + "logps/chosen": -47.75170135498047, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.53636169433594, + "loss": 1.3815, + "margin_dpo/margin_mean": 0.0518706738948822, + "margin_dpo/margin_std": 0.3616672158241272, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.024654388427734375, + "KL/mean": -0.01638704538345337, + "KL/rejected_KL_mean": -0.057422637939453125, + "KL/std": 0.22416919469833374, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08206850290298462, + "fcm_dpo/q_t": 0.4979493319988251, + "grad_norm": 73.6179428100586, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.5047850608825684, + "logits/rejected": -0.45597630739212036, + "logps/chosen": -70.18070983886719, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.81501770019531, + "loss": 1.3784, + "margin_dpo/margin_mean": 0.08206871151924133, + "margin_dpo/margin_std": 0.34193331003189087, + "step": 19 + }, + { + "KL/chosen_KL_mean": -0.0042552947998046875, + "KL/mean": -0.03660614788532257, + "KL/rejected_KL_mean": -0.06895065307617188, + "KL/std": 0.2531750798225403, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06470224261283875, + "fcm_dpo/q_t": 0.498382031917572, + "grad_norm": 73.99443817138672, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5425054430961609, + "logits/rejected": -0.4851202964782715, + "logps/chosen": -50.8074951171875, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.89230346679688, + "loss": 1.3802, + "margin_dpo/margin_mean": 0.0647021234035492, + "margin_dpo/margin_std": 0.3650708794593811, + "step": 20 + }, + { + "KL/chosen_KL_mean": -0.030855178833007812, + "KL/mean": -0.07397759705781937, + "KL/rejected_KL_mean": -0.11709976196289062, + "KL/std": 0.2903878688812256, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08624540269374847, + "fcm_dpo/q_t": 0.4978443384170532, + "grad_norm": 77.94503021240234, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.516939103603363, + "logits/rejected": -0.49480289220809937, + "logps/chosen": -50.0938720703125, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.98589324951172, + "loss": 1.3781, + "margin_dpo/margin_mean": 0.0862455815076828, + "margin_dpo/margin_std": 0.40781253576278687, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.04010009765625, + "KL/mean": -0.07934239506721497, + "KL/rejected_KL_mean": -0.19878387451171875, + "KL/std": 0.2584654688835144, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23888826370239258, + "fcm_dpo/q_t": 0.49403008818626404, + "grad_norm": 84.08679962158203, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.4520847797393799, + "logits/rejected": -0.4066694378852844, + "logps/chosen": -59.01753616333008, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.70346069335938, + "loss": 1.3628, + "margin_dpo/margin_mean": 0.23888859152793884, + "margin_dpo/margin_std": 0.3350480794906616, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.0505218505859375, + "KL/mean": -0.07051441073417664, + "KL/rejected_KL_mean": -0.19155120849609375, + "KL/std": 0.3092515468597412, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24207347631454468, + "fcm_dpo/q_t": 0.4939531087875366, + "grad_norm": 80.46525573730469, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5085234045982361, + "logits/rejected": -0.4870685040950775, + "logps/chosen": -60.02717590332031, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.33110046386719, + "loss": 1.3627, + "margin_dpo/margin_mean": 0.2420736849308014, + "margin_dpo/margin_std": 0.4334256947040558, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.051456451416015625, + "KL/mean": -0.06804545968770981, + "KL/rejected_KL_mean": -0.18755340576171875, + "KL/std": 0.3063574433326721, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.23900540173053741, + "fcm_dpo/q_t": 0.4940268397331238, + "grad_norm": 79.16189575195312, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5088186264038086, + "logits/rejected": -0.4924226701259613, + "logps/chosen": -44.23957824707031, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.3127670288086, + "loss": 1.363, + "margin_dpo/margin_mean": 0.23900574445724487, + "margin_dpo/margin_std": 0.4075234532356262, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.011472702026367188, + "KL/mean": -0.1012650728225708, + "KL/rejected_KL_mean": -0.21400070190429688, + "KL/std": 0.37399694323539734, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22547590732574463, + "fcm_dpo/q_t": 0.4943666458129883, + "grad_norm": 73.96826934814453, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.5034818053245544, + "logits/rejected": -0.47385311126708984, + "logps/chosen": -52.52558135986328, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.55619049072266, + "loss": 1.3644, + "margin_dpo/margin_mean": 0.22547617554664612, + "margin_dpo/margin_std": 0.440443217754364, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.06828689575195312, + "KL/mean": -0.12919658422470093, + "KL/rejected_KL_mean": -0.32668304443359375, + "KL/std": 0.43470126390457153, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3949729800224304, + "fcm_dpo/q_t": 0.49013736844062805, + "grad_norm": 86.56037139892578, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5416176319122314, + "logits/rejected": -0.510056734085083, + "logps/chosen": -53.854522705078125, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.6864013671875, + "loss": 1.3479, + "margin_dpo/margin_mean": 0.3949730694293976, + "margin_dpo/margin_std": 0.5318588614463806, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.13717269897460938, + "KL/mean": -0.1322329044342041, + "KL/rejected_KL_mean": -0.4016380310058594, + "KL/std": 0.46932661533355713, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5388086438179016, + "fcm_dpo/q_t": 0.48654913902282715, + "grad_norm": 92.75858306884766, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.5484895706176758, + "logits/rejected": -0.5124675035476685, + "logps/chosen": -42.761356353759766, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.12583923339844, + "loss": 1.334, + "margin_dpo/margin_mean": 0.5388088822364807, + "margin_dpo/margin_std": 0.5669536590576172, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.014291763305664062, + "KL/mean": -0.14200696349143982, + "KL/rejected_KL_mean": -0.26972198486328125, + "KL/std": 0.38193371891975403, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2554277181625366, + "fcm_dpo/q_t": 0.49362027645111084, + "grad_norm": 75.72992706298828, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.524590015411377, + "logits/rejected": -0.4710499346256256, + "logps/chosen": -60.57079315185547, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.67083740234375, + "loss": 1.3617, + "margin_dpo/margin_mean": 0.2554280161857605, + "margin_dpo/margin_std": 0.5491482019424438, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.12394332885742188, + "KL/mean": -0.17407271265983582, + "KL/rejected_KL_mean": -0.4720878601074219, + "KL/std": 0.5238059759140015, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.596030056476593, + "fcm_dpo/q_t": 0.4851195514202118, + "grad_norm": 91.14883422851562, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.5426310896873474, + "logits/rejected": -0.4955989420413971, + "logps/chosen": -57.683841705322266, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.86643981933594, + "loss": 1.3285, + "margin_dpo/margin_mean": 0.5960307121276855, + "margin_dpo/margin_std": 0.5940129160881042, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.14072608947753906, + "KL/mean": -0.1971840113401413, + "KL/rejected_KL_mean": -0.5350990295410156, + "KL/std": 0.5936540961265564, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6758224964141846, + "fcm_dpo/q_t": 0.48313936591148376, + "grad_norm": 87.16856384277344, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.49958544969558716, + "logits/rejected": -0.46898987889289856, + "logps/chosen": -52.4366455078125, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -99.02430725097656, + "loss": 1.321, + "margin_dpo/margin_mean": 0.675823450088501, + "margin_dpo/margin_std": 0.671462893486023, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.12239265441894531, + "KL/mean": -0.10819974541664124, + "KL/rejected_KL_mean": -0.33879852294921875, + "KL/std": 0.5320932865142822, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.46118801832199097, + "fcm_dpo/q_t": 0.48849618434906006, + "grad_norm": 67.80732727050781, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.5115536451339722, + "logits/rejected": -0.4674157500267029, + "logps/chosen": -63.68452835083008, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.23280334472656, + "loss": 1.342, + "margin_dpo/margin_mean": 0.4611876904964447, + "margin_dpo/margin_std": 0.6903836727142334, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.23033905029296875, + "KL/mean": -0.18237151205539703, + "KL/rejected_KL_mean": -0.5950851440429688, + "KL/std": 0.7562414407730103, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8254227638244629, + "fcm_dpo/q_t": 0.4794422388076782, + "grad_norm": 81.30719757080078, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.5320795774459839, + "logits/rejected": -0.49193692207336426, + "logps/chosen": -62.509185791015625, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.91258239746094, + "loss": 1.3076, + "margin_dpo/margin_mean": 0.8254228830337524, + "margin_dpo/margin_std": 0.9202225208282471, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.08339118957519531, + "KL/mean": -0.1864042431116104, + "KL/rejected_KL_mean": -0.4561958312988281, + "KL/std": 0.5232335329055786, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5395828485488892, + "fcm_dpo/q_t": 0.48652732372283936, + "grad_norm": 72.66676330566406, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.5073741674423218, + "logits/rejected": -0.4817469120025635, + "logps/chosen": -53.177581787109375, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.3413314819336, + "loss": 1.334, + "margin_dpo/margin_mean": 0.5395834445953369, + "margin_dpo/margin_std": 0.5871918201446533, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.1102294921875, + "KL/mean": -0.2915915846824646, + "KL/rejected_KL_mean": -0.69342041015625, + "KL/std": 0.7285856604576111, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8036417961120605, + "fcm_dpo/q_t": 0.479966402053833, + "grad_norm": 77.63619995117188, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.49800926446914673, + "logits/rejected": -0.4805489480495453, + "logps/chosen": -50.70709991455078, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.61526489257812, + "loss": 1.3094, + "margin_dpo/margin_mean": 0.8036414384841919, + "margin_dpo/margin_std": 0.8560863137245178, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.1316852569580078, + "KL/mean": -0.4846244752407074, + "KL/rejected_KL_mean": -1.1009368896484375, + "KL/std": 1.0043591260910034, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2326196432113647, + "fcm_dpo/q_t": 0.46936315298080444, + "grad_norm": 83.03388214111328, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.5253646373748779, + "logits/rejected": -0.4884377419948578, + "logps/chosen": -50.89280700683594, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.92536926269531, + "loss": 1.2701, + "margin_dpo/margin_mean": 1.2326194047927856, + "margin_dpo/margin_std": 1.1317143440246582, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.047260284423828125, + "KL/mean": -0.5339961647987366, + "KL/rejected_KL_mean": -1.1152496337890625, + "KL/std": 1.0842790603637695, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1625124216079712, + "fcm_dpo/q_t": 0.4710965156555176, + "grad_norm": 73.23500061035156, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.570366621017456, + "logits/rejected": -0.5346090793609619, + "logps/chosen": -51.94422912597656, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.15587615966797, + "loss": 1.2773, + "margin_dpo/margin_mean": 1.1625115871429443, + "margin_dpo/margin_std": 1.1846086978912354, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.023740768432617188, + "KL/mean": -0.5017684698104858, + "KL/rejected_KL_mean": -0.97979736328125, + "KL/std": 1.039499044418335, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.9560521245002747, + "fcm_dpo/q_t": 0.4763021469116211, + "grad_norm": 62.15679931640625, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5129455327987671, + "logits/rejected": -0.46945592761039734, + "logps/chosen": -62.830848693847656, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.87487030029297, + "loss": 1.298, + "margin_dpo/margin_mean": 0.9560526609420776, + "margin_dpo/margin_std": 1.37896728515625, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1662731170654297, + "KL/mean": -0.5158271193504333, + "KL/rejected_KL_mean": -1.1979255676269531, + "KL/std": 1.333385705947876, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3642008304595947, + "fcm_dpo/q_t": 0.46630561351776123, + "grad_norm": 69.56143951416016, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5130950212478638, + "logits/rejected": -0.4803865849971771, + "logps/chosen": -48.224246978759766, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.11036682128906, + "loss": 1.2609, + "margin_dpo/margin_mean": 1.364201307296753, + "margin_dpo/margin_std": 1.6021790504455566, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.10568809509277344, + "KL/mean": -0.7288994193077087, + "KL/rejected_KL_mean": -1.5634841918945312, + "KL/std": 1.3381998538970947, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6691735982894897, + "fcm_dpo/q_t": 0.45862340927124023, + "grad_norm": 73.14259338378906, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5661517381668091, + "logits/rejected": -0.5268115997314453, + "logps/chosen": -50.64478302001953, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.13299560546875, + "loss": 1.2318, + "margin_dpo/margin_mean": 1.6691731214523315, + "margin_dpo/margin_std": 1.4717893600463867, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.1655292510986328, + "KL/mean": -0.6022967100143433, + "KL/rejected_KL_mean": -1.3701248168945312, + "KL/std": 1.372552514076233, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.535652756690979, + "fcm_dpo/q_t": 0.4620806872844696, + "grad_norm": 60.41756820678711, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5125629901885986, + "logits/rejected": -0.4821798801422119, + "logps/chosen": -57.81953811645508, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.67019653320312, + "loss": 1.246, + "margin_dpo/margin_mean": 1.5356531143188477, + "margin_dpo/margin_std": 1.6803004741668701, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.018140792846679688, + "KL/mean": -0.9324614405632019, + "KL/rejected_KL_mean": -1.8830604553222656, + "KL/std": 1.865638256072998, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.9012001752853394, + "fcm_dpo/q_t": 0.45320603251457214, + "grad_norm": 68.2044906616211, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5240455865859985, + "logits/rejected": -0.4866907000541687, + "logps/chosen": -62.67767333984375, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.90658569335938, + "loss": 1.2151, + "margin_dpo/margin_mean": 1.9012011289596558, + "margin_dpo/margin_std": 1.967465877532959, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.22827911376953125, + "KL/mean": -1.0795395374298096, + "KL/rejected_KL_mean": -2.3873558044433594, + "KL/std": 2.1814446449279785, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.615635871887207, + "fcm_dpo/q_t": 0.436246395111084, + "grad_norm": 78.9505844116211, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5361723899841309, + "logits/rejected": -0.4895145893096924, + "logps/chosen": -58.7381477355957, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.29573059082031, + "loss": 1.1563, + "margin_dpo/margin_mean": 2.615636110305786, + "margin_dpo/margin_std": 2.4361557960510254, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.48670196533203125, + "KL/mean": -0.7337930798530579, + "KL/rejected_KL_mean": -1.9542922973632812, + "KL/std": 1.8439135551452637, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.4409918785095215, + "fcm_dpo/q_t": 0.4401203989982605, + "grad_norm": 71.73788452148438, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5428401827812195, + "logits/rejected": -0.5178759098052979, + "logps/chosen": -53.66929626464844, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.43447875976562, + "loss": 1.1676, + "margin_dpo/margin_mean": 2.4409918785095215, + "margin_dpo/margin_std": 2.0570061206817627, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.23327255249023438, + "KL/mean": -1.1817584037780762, + "KL/rejected_KL_mean": -2.5967864990234375, + "KL/std": 2.2948923110961914, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.1010427176952362, + "fcm_dpo/delta": 0.05186600610613823, + "fcm_dpo/margin": 2.830059289932251, + "fcm_dpo/q_t": 0.4304935932159424, + "grad_norm": 79.19278717041016, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.45061954855918884, + "logits/rejected": -0.4311766028404236, + "logps/chosen": -49.8452262878418, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.38054656982422, + "loss": 1.1337, + "margin_dpo/margin_mean": 2.83005952835083, + "margin_dpo/margin_std": 2.295668363571167, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.13261985778808594, + "KL/mean": -0.9955662488937378, + "KL/rejected_KL_mean": -2.1237525939941406, + "KL/std": 2.05623722076416, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.1010427176952362, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.2563743591308594, + "fcm_dpo/q_t": 0.44458091259002686, + "grad_norm": 63.24766159057617, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.4729717969894409, + "logits/rejected": -0.46019092202186584, + "logps/chosen": -48.282310485839844, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.0601806640625, + "loss": 1.1877, + "margin_dpo/margin_mean": 2.256373882293701, + "margin_dpo/margin_std": 2.5547118186950684, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.19269752502441406, + "KL/mean": -1.2841308116912842, + "KL/rejected_KL_mean": -2.7609596252441406, + "KL/std": 2.6203222274780273, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.1014559417963028, + "fcm_dpo/delta": 0.04072940722107887, + "fcm_dpo/margin": 2.9536566734313965, + "fcm_dpo/q_t": 0.4284740090370178, + "grad_norm": 70.56751251220703, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5397986173629761, + "logits/rejected": -0.489362895488739, + "logps/chosen": -55.80672836303711, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.41354370117188, + "loss": 1.1343, + "margin_dpo/margin_mean": 2.9536566734313965, + "margin_dpo/margin_std": 3.2299246788024902, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.34847259521484375, + "KL/mean": -1.144223928451538, + "KL/rejected_KL_mean": -2.636920928955078, + "KL/std": 2.6396236419677734, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10205230116844177, + "fcm_dpo/delta": 0.017946049571037292, + "fcm_dpo/margin": 2.985393524169922, + "fcm_dpo/q_t": 0.4262898564338684, + "grad_norm": 66.21025085449219, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5646312832832336, + "logits/rejected": -0.5113937258720398, + "logps/chosen": -57.577606201171875, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.31613159179688, + "loss": 1.1244, + "margin_dpo/margin_mean": 2.98539400100708, + "margin_dpo/margin_std": 2.747206926345825, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.03780937194824219, + "KL/mean": -1.5395220518112183, + "KL/rejected_KL_mean": -3.1168556213378906, + "KL/std": 2.745673179626465, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.10240267217159271, + "fcm_dpo/delta": 0.008171799592673779, + "fcm_dpo/margin": 3.1546638011932373, + "fcm_dpo/q_t": 0.42221391201019287, + "grad_norm": 72.73230743408203, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5809261202812195, + "logits/rejected": -0.5220905542373657, + "logps/chosen": -57.150264739990234, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.13345336914062, + "loss": 1.116, + "margin_dpo/margin_mean": 3.154663562774658, + "margin_dpo/margin_std": 3.0134170055389404, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.2747459411621094, + "KL/mean": -1.6129170656204224, + "KL/rejected_KL_mean": -3.5005836486816406, + "KL/std": 3.4164953231811523, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.10293182730674744, + "fcm_dpo/delta": 0.011580899357795715, + "fcm_dpo/margin": 3.775324583053589, + "fcm_dpo/q_t": 0.4091089963912964, + "grad_norm": 62.16366195678711, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5313198566436768, + "logits/rejected": -0.4705466032028198, + "logps/chosen": -61.4105224609375, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.26805877685547, + "loss": 1.0755, + "margin_dpo/margin_mean": 3.775324583053589, + "margin_dpo/margin_std": 3.9806838035583496, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.1489086151123047, + "KL/mean": -2.2461960315704346, + "KL/rejected_KL_mean": -4.343479156494141, + "KL/std": 3.5566017627716064, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10210566222667694, + "fcm_dpo/delta": -0.0295924860984087, + "fcm_dpo/margin": 4.19456672668457, + "fcm_dpo/q_t": 0.39893922209739685, + "grad_norm": 63.00365447998047, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5612349510192871, + "logits/rejected": -0.5258386731147766, + "logps/chosen": -58.873046875, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.70162963867188, + "loss": 1.0432, + "margin_dpo/margin_mean": 4.1945672035217285, + "margin_dpo/margin_std": 4.0588274002075195, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2848930358886719, + "KL/mean": -2.314423084259033, + "KL/rejected_KL_mean": -4.343959808349609, + "KL/std": 4.194530963897705, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10142997652292252, + "fcm_dpo/delta": -0.01260334625840187, + "fcm_dpo/margin": 4.05906343460083, + "fcm_dpo/q_t": 0.4068337678909302, + "grad_norm": 52.816749572753906, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.511359691619873, + "logits/rejected": -0.4778144955635071, + "logps/chosen": -61.65856170654297, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.34596252441406, + "loss": 1.0802, + "margin_dpo/margin_mean": 4.059062957763672, + "margin_dpo/margin_std": 5.207043647766113, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.3401203155517578, + "KL/mean": -2.740550994873047, + "KL/rejected_KL_mean": -5.82122802734375, + "KL/std": 4.815142631530762, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.09850598871707916, + "fcm_dpo/delta": -0.22030740976333618, + "fcm_dpo/margin": 6.161346435546875, + "fcm_dpo/q_t": 0.35945144295692444, + "grad_norm": 58.77971649169922, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5420224666595459, + "logits/rejected": -0.48554420471191406, + "logps/chosen": -51.99723815917969, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.7951431274414, + "loss": 0.9262, + "margin_dpo/margin_mean": 6.161346435546875, + "margin_dpo/margin_std": 5.183620929718018, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.1926727294921875, + "KL/mean": -3.3358712196350098, + "KL/rejected_KL_mean": -6.479068756103516, + "KL/std": 5.467696666717529, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.09512735903263092, + "fcm_dpo/delta": -0.21018867194652557, + "fcm_dpo/margin": 6.286403179168701, + "fcm_dpo/q_t": 0.36875689029693604, + "grad_norm": 56.959957122802734, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6224546432495117, + "logits/rejected": -0.6011621356010437, + "logps/chosen": -53.50732421875, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.26266479492188, + "loss": 0.9622, + "margin_dpo/margin_mean": 6.286401748657227, + "margin_dpo/margin_std": 6.2769670486450195, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.4639778137207031, + "KL/mean": -3.161789894104004, + "KL/rejected_KL_mean": -5.859600067138672, + "KL/std": 5.118355751037598, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.09215639531612396, + "fcm_dpo/delta": -0.10220731794834137, + "fcm_dpo/margin": 5.395620346069336, + "fcm_dpo/q_t": 0.38467535376548767, + "grad_norm": 52.45750045776367, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.593686580657959, + "logits/rejected": -0.5403316020965576, + "logps/chosen": -51.15264129638672, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.57499694824219, + "loss": 0.9995, + "margin_dpo/margin_mean": 5.3956193923950195, + "margin_dpo/margin_std": 5.11165714263916, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.9617080688476562, + "KL/mean": -4.2060651779174805, + "KL/rejected_KL_mean": -7.450420379638672, + "KL/std": 6.193329334259033, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.08927452564239502, + "fcm_dpo/delta": -0.18988440930843353, + "fcm_dpo/margin": 6.488712787628174, + "fcm_dpo/q_t": 0.3716598153114319, + "grad_norm": 50.36177062988281, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6261263489723206, + "logits/rejected": -0.5618505477905273, + "logps/chosen": -63.576942443847656, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.44392395019531, + "loss": 0.9866, + "margin_dpo/margin_mean": 6.488712310791016, + "margin_dpo/margin_std": 7.48520565032959, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.6866493225097656, + "KL/mean": -3.781385898590088, + "KL/rejected_KL_mean": -6.876125335693359, + "KL/std": 6.0763092041015625, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.08633197098970413, + "fcm_dpo/delta": -0.14181526005268097, + "fcm_dpo/margin": 6.189474105834961, + "fcm_dpo/q_t": 0.38100165128707886, + "grad_norm": 44.8555793762207, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.6169860363006592, + "logits/rejected": -0.5749229192733765, + "logps/chosen": -58.61937713623047, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.05056762695312, + "loss": 1.013, + "margin_dpo/margin_mean": 6.189474105834961, + "margin_dpo/margin_std": 7.407299041748047, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7061119079589844, + "KL/mean": -4.236698150634766, + "KL/rejected_KL_mean": -7.767284393310547, + "KL/std": 6.019701957702637, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.08343879133462906, + "fcm_dpo/delta": -0.20055079460144043, + "fcm_dpo/margin": 7.06117057800293, + "fcm_dpo/q_t": 0.3642050623893738, + "grad_norm": 49.13716506958008, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5706203579902649, + "logits/rejected": -0.5421047210693359, + "logps/chosen": -71.20140075683594, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.3327407836914, + "loss": 0.952, + "margin_dpo/margin_mean": 7.06117057800293, + "margin_dpo/margin_std": 6.6234002113342285, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.0463066101074219, + "KL/mean": -4.894023418426514, + "KL/rejected_KL_mean": -8.741737365722656, + "KL/std": 6.796070098876953, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.07982903718948364, + "fcm_dpo/delta": -0.22777986526489258, + "fcm_dpo/margin": 7.695425987243652, + "fcm_dpo/q_t": 0.36338773369789124, + "grad_norm": 51.69680404663086, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.6090884804725647, + "logits/rejected": -0.5345608592033386, + "logps/chosen": -63.17924499511719, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.3590316772461, + "loss": 0.956, + "margin_dpo/margin_mean": 7.695425987243652, + "margin_dpo/margin_std": 8.11735725402832, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.5256767272949219, + "KL/mean": -5.767217636108398, + "KL/rejected_KL_mean": -10.008758544921875, + "KL/std": 7.521817207336426, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.07504991441965103, + "fcm_dpo/delta": -0.256174236536026, + "fcm_dpo/margin": 8.483085632324219, + "fcm_dpo/q_t": 0.3579879105091095, + "grad_norm": 48.301658630371094, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6136581897735596, + "logits/rejected": -0.5725275278091431, + "logps/chosen": -53.45820236206055, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.89395904541016, + "loss": 0.9406, + "margin_dpo/margin_mean": 8.483086585998535, + "margin_dpo/margin_std": 8.631021499633789, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.7287673950195312, + "KL/mean": -6.115081787109375, + "KL/rejected_KL_mean": -9.501392364501953, + "KL/std": 6.875812530517578, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.07330524921417236, + "fcm_dpo/delta": -0.10204505175352097, + "fcm_dpo/margin": 6.77262020111084, + "fcm_dpo/q_t": 0.3847336769104004, + "grad_norm": 51.394508361816406, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.613247811794281, + "logits/rejected": -0.5534546971321106, + "logps/chosen": -63.670955657958984, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.89479064941406, + "loss": 1.0269, + "margin_dpo/margin_mean": 6.772619724273682, + "margin_dpo/margin_std": 7.818716049194336, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.5299663543701172, + "KL/mean": -5.733615875244141, + "KL/rejected_KL_mean": -9.937267303466797, + "KL/std": 9.215574264526367, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.07065397500991821, + "fcm_dpo/delta": -0.20743195712566376, + "fcm_dpo/margin": 8.407304763793945, + "fcm_dpo/q_t": 0.3753771185874939, + "grad_norm": 43.15869140625, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6083458662033081, + "logits/rejected": -0.5743233561515808, + "logps/chosen": -62.163490295410156, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.78976440429688, + "loss": 1.0042, + "margin_dpo/margin_mean": 8.407305717468262, + "margin_dpo/margin_std": 11.433292388916016, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.7452468872070312, + "KL/mean": -4.873937606811523, + "KL/rejected_KL_mean": -8.00262451171875, + "KL/std": 6.709729194641113, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.07000763714313507, + "fcm_dpo/delta": -0.039910588413476944, + "fcm_dpo/margin": 6.257376194000244, + "fcm_dpo/q_t": 0.40113139152526855, + "grad_norm": 42.04322814941406, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.6177521347999573, + "logits/rejected": -0.5839763879776001, + "logps/chosen": -57.896018981933594, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.56881713867188, + "loss": 1.0728, + "margin_dpo/margin_mean": 6.257376670837402, + "margin_dpo/margin_std": 8.252243041992188, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.845205307006836, + "KL/mean": -7.078099727630615, + "KL/rejected_KL_mean": -11.310993194580078, + "KL/std": 8.602740287780762, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.06737160682678223, + "fcm_dpo/delta": -0.18241550028324127, + "fcm_dpo/margin": 8.465780258178711, + "fcm_dpo/q_t": 0.36989033222198486, + "grad_norm": 44.06743621826172, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6237053871154785, + "logits/rejected": -0.5804514288902283, + "logps/chosen": -75.99259948730469, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -108.92105102539062, + "loss": 0.9799, + "margin_dpo/margin_mean": 8.465781211853027, + "margin_dpo/margin_std": 9.02548599243164, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.0796890258789062, + "KL/mean": -6.119532585144043, + "KL/rejected_KL_mean": -11.159374237060547, + "KL/std": 9.254194259643555, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.06391540169715881, + "fcm_dpo/delta": -0.2655850946903229, + "fcm_dpo/margin": 10.079679489135742, + "fcm_dpo/q_t": 0.35841235518455505, + "grad_norm": 39.89701461791992, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.5818270444869995, + "logits/rejected": -0.549256443977356, + "logps/chosen": -55.07828903198242, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.6895751953125, + "loss": 0.9546, + "margin_dpo/margin_mean": 10.079680442810059, + "margin_dpo/margin_std": 11.321163177490234, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.4329071044921875, + "KL/mean": -8.504103660583496, + "KL/rejected_KL_mean": -13.575305938720703, + "KL/std": 10.478982925415039, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.06112462282180786, + "fcm_dpo/delta": -0.23643510043621063, + "fcm_dpo/margin": 10.142398834228516, + "fcm_dpo/q_t": 0.36100542545318604, + "grad_norm": 40.643394470214844, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6882668137550354, + "logits/rejected": -0.675888180732727, + "logps/chosen": -68.26890563964844, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.52176666259766, + "loss": 0.9702, + "margin_dpo/margin_mean": 10.142398834228516, + "margin_dpo/margin_std": 11.497425079345703, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.2129268646240234, + "KL/mean": -7.869783401489258, + "KL/rejected_KL_mean": -12.52663803100586, + "KL/std": 9.454305648803711, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.05889023840427399, + "fcm_dpo/delta": -0.15918777883052826, + "fcm_dpo/margin": 9.313708305358887, + "fcm_dpo/q_t": 0.37943077087402344, + "grad_norm": 36.45185089111328, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.6642798185348511, + "logits/rejected": -0.632885754108429, + "logps/chosen": -54.65645217895508, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -88.16293334960938, + "loss": 1.0071, + "margin_dpo/margin_mean": 9.313708305358887, + "margin_dpo/margin_std": 11.38536262512207, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.8387413024902344, + "KL/mean": -7.603968143463135, + "KL/rejected_KL_mean": -12.369194030761719, + "KL/std": 9.500910758972168, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.05760319158434868, + "fcm_dpo/delta": -0.1573733240365982, + "fcm_dpo/margin": 9.530448913574219, + "fcm_dpo/q_t": 0.37900108098983765, + "grad_norm": 36.39604949951172, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6034436225891113, + "logits/rejected": -0.5606616735458374, + "logps/chosen": -62.17954635620117, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -85.15647888183594, + "loss": 1.0047, + "margin_dpo/margin_mean": 9.530448913574219, + "margin_dpo/margin_std": 11.51986026763916, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.4067440032958984, + "KL/mean": -7.966508865356445, + "KL/rejected_KL_mean": -12.52627182006836, + "KL/std": 8.551124572753906, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.056131426244974136, + "fcm_dpo/delta": -0.11776769906282425, + "fcm_dpo/margin": 9.119524002075195, + "fcm_dpo/q_t": 0.3830665946006775, + "grad_norm": 35.99039840698242, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6368460655212402, + "logits/rejected": -0.5754811763763428, + "logps/chosen": -68.61257934570312, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -89.7335205078125, + "loss": 1.005, + "margin_dpo/margin_mean": 9.119524002075195, + "margin_dpo/margin_std": 9.659480094909668, + "step": 68 + }, + { + "KL/chosen_KL_mean": -4.0277862548828125, + "KL/mean": -9.794254302978516, + "KL/rejected_KL_mean": -15.560726165771484, + "KL/std": 10.441646575927734, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.0535246804356575, + "fcm_dpo/delta": -0.2331288605928421, + "fcm_dpo/margin": 11.532941818237305, + "fcm_dpo/q_t": 0.3619251251220703, + "grad_norm": 38.06724548339844, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.624565839767456, + "logits/rejected": -0.6012846231460571, + "logps/chosen": -63.84702682495117, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -118.9495849609375, + "loss": 0.9436, + "margin_dpo/margin_mean": 11.532941818237305, + "margin_dpo/margin_std": 11.566747665405273, + "step": 69 + }, + { + "KL/chosen_KL_mean": -5.952953338623047, + "KL/mean": -12.226800918579102, + "KL/rejected_KL_mean": -18.500648498535156, + "KL/std": 13.060888290405273, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.051290594041347504, + "fcm_dpo/delta": -0.25971972942352295, + "fcm_dpo/margin": 12.54769229888916, + "fcm_dpo/q_t": 0.35949984192848206, + "grad_norm": 38.32661819458008, + "learning_rate": 5e-07, + "logits/chosen": -0.6319253444671631, + "logits/rejected": -0.5981800556182861, + "logps/chosen": -67.88359069824219, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -109.56143188476562, + "loss": 0.9516, + "margin_dpo/margin_mean": 12.547693252563477, + "margin_dpo/margin_std": 13.876760482788086, + "step": 70 + }, + { + "KL/chosen_KL_mean": -5.886379241943359, + "KL/mean": -13.411661148071289, + "KL/rejected_KL_mean": -20.93695068359375, + "KL/std": 13.92454719543457, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.0477573461830616, + "fcm_dpo/delta": -0.3450050354003906, + "fcm_dpo/margin": 15.050573348999023, + "fcm_dpo/q_t": 0.3452618718147278, + "grad_norm": 35.40090560913086, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6766068935394287, + "logits/rejected": -0.6392951607704163, + "logps/chosen": -67.63671875, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -118.27357482910156, + "loss": 0.9002, + "margin_dpo/margin_mean": 15.050573348999023, + "margin_dpo/margin_std": 15.189107894897461, + "step": 71 + }, + { + "KL/chosen_KL_mean": -6.88047981262207, + "KL/mean": -14.686164855957031, + "KL/rejected_KL_mean": -22.491851806640625, + "KL/std": 14.913492202758789, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.044771067798137665, + "fcm_dpo/delta": -0.32236674427986145, + "fcm_dpo/margin": 15.611374855041504, + "fcm_dpo/q_t": 0.3481115400791168, + "grad_norm": 41.994754791259766, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6647568941116333, + "logits/rejected": -0.6286379098892212, + "logps/chosen": -72.93389129638672, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -117.77883911132812, + "loss": 0.9317, + "margin_dpo/margin_mean": 15.611373901367188, + "margin_dpo/margin_std": 17.554119110107422, + "step": 72 + }, + { + "KL/chosen_KL_mean": -8.87645149230957, + "KL/mean": -16.725805282592773, + "KL/rejected_KL_mean": -24.575157165527344, + "KL/std": 19.394638061523438, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.04213154315948486, + "fcm_dpo/delta": -0.28175461292266846, + "fcm_dpo/margin": 15.698704719543457, + "fcm_dpo/q_t": 0.36960694193840027, + "grad_norm": 37.33930206298828, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6842408776283264, + "logits/rejected": -0.6434606313705444, + "logps/chosen": -75.13272094726562, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -115.03129577636719, + "loss": 1.0376, + "margin_dpo/margin_mean": 15.698704719543457, + "margin_dpo/margin_std": 24.332979202270508, + "step": 73 + }, + { + "KL/chosen_KL_mean": -9.426824569702148, + "KL/mean": -18.018756866455078, + "KL/rejected_KL_mean": -26.61068344116211, + "KL/std": 21.296592712402344, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.03975830227136612, + "fcm_dpo/delta": -0.30507808923721313, + "fcm_dpo/margin": 17.183855056762695, + "fcm_dpo/q_t": 0.3635675609111786, + "grad_norm": 35.87363052368164, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6560777425765991, + "logits/rejected": -0.6484391689300537, + "logps/chosen": -62.851707458496094, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -122.5576171875, + "loss": 0.9964, + "margin_dpo/margin_mean": 17.183853149414062, + "margin_dpo/margin_std": 26.46825408935547, + "step": 74 + }, + { + "KL/chosen_KL_mean": -9.193737030029297, + "KL/mean": -21.270729064941406, + "KL/rejected_KL_mean": -33.34771728515625, + "KL/std": 21.801738739013672, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.03657674044370651, + "fcm_dpo/delta": -0.5276959538459778, + "fcm_dpo/margin": 24.15398406982422, + "fcm_dpo/q_t": 0.3196951746940613, + "grad_norm": 30.990449905395508, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6813095211982727, + "logits/rejected": -0.6741442680358887, + "logps/chosen": -61.055397033691406, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -144.60169982910156, + "loss": 0.8461, + "margin_dpo/margin_mean": 24.15398406982422, + "margin_dpo/margin_std": 24.76327133178711, + "step": 75 + }, + { + "KL/chosen_KL_mean": -11.104537963867188, + "KL/mean": -18.71759796142578, + "KL/rejected_KL_mean": -26.33065414428711, + "KL/std": 17.72130012512207, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.03464771807193756, + "fcm_dpo/delta": -0.13456206023693085, + "fcm_dpo/margin": 15.226126670837402, + "fcm_dpo/q_t": 0.38007593154907227, + "grad_norm": 32.52539825439453, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.7033958435058594, + "logits/rejected": -0.662422239780426, + "logps/chosen": -64.37057495117188, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -104.54728698730469, + "loss": 1.0266, + "margin_dpo/margin_mean": 15.226127624511719, + "margin_dpo/margin_std": 19.432636260986328, + "step": 76 + }, + { + "KL/chosen_KL_mean": -9.855213165283203, + "KL/mean": -22.263296127319336, + "KL/rejected_KL_mean": -34.671382904052734, + "KL/std": 24.060508728027344, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.0320308655500412, + "fcm_dpo/delta": -0.4329489469528198, + "fcm_dpo/margin": 24.816164016723633, + "fcm_dpo/q_t": 0.32857275009155273, + "grad_norm": 32.527626037597656, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6911159753799438, + "logits/rejected": -0.6543018817901611, + "logps/chosen": -67.95188903808594, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -128.4449920654297, + "loss": 0.8929, + "margin_dpo/margin_mean": 24.816162109375, + "margin_dpo/margin_std": 26.715883255004883, + "step": 77 + }, + { + "KL/chosen_KL_mean": -10.004676818847656, + "KL/mean": -19.24388885498047, + "KL/rejected_KL_mean": -28.48310089111328, + "KL/std": 19.375869750976562, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.0307869091629982, + "fcm_dpo/delta": -0.17875218391418457, + "fcm_dpo/margin": 18.478416442871094, + "fcm_dpo/q_t": 0.37511003017425537, + "grad_norm": 29.487205505371094, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6501991152763367, + "logits/rejected": -0.6261272430419922, + "logps/chosen": -65.61846160888672, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -113.41746520996094, + "loss": 1.0008, + "margin_dpo/margin_mean": 18.478416442871094, + "margin_dpo/margin_std": 22.34000015258789, + "step": 78 + }, + { + "KL/chosen_KL_mean": -10.239093780517578, + "KL/mean": -20.427974700927734, + "KL/rejected_KL_mean": -30.61685562133789, + "KL/std": 22.000255584716797, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.029458634555339813, + "fcm_dpo/delta": -0.2132250964641571, + "fcm_dpo/margin": 20.377758026123047, + "fcm_dpo/q_t": 0.3738274574279785, + "grad_norm": 26.049671173095703, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6516468524932861, + "logits/rejected": -0.621735692024231, + "logps/chosen": -65.68958282470703, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -118.26441955566406, + "loss": 1.0065, + "margin_dpo/margin_mean": 20.377758026123047, + "margin_dpo/margin_std": 26.898948669433594, + "step": 79 + }, + { + "KL/chosen_KL_mean": -13.422794342041016, + "KL/mean": -22.96697235107422, + "KL/rejected_KL_mean": -32.511146545410156, + "KL/std": 23.346824645996094, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.028284694999456406, + "fcm_dpo/delta": -0.1503477245569229, + "fcm_dpo/margin": 19.08835220336914, + "fcm_dpo/q_t": 0.3875836730003357, + "grad_norm": 28.567445755004883, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7152136564254761, + "logits/rejected": -0.6735327243804932, + "logps/chosen": -71.94208526611328, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -120.05864715576172, + "loss": 1.0492, + "margin_dpo/margin_mean": 19.08835220336914, + "margin_dpo/margin_std": 27.130773544311523, + "step": 80 + }, + { + "KL/chosen_KL_mean": -12.539752960205078, + "KL/mean": -25.574947357177734, + "KL/rejected_KL_mean": -38.610145568847656, + "KL/std": 27.4758243560791, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.026796914637088776, + "fcm_dpo/delta": -0.32385769486427307, + "fcm_dpo/margin": 26.070392608642578, + "fcm_dpo/q_t": 0.35455310344696045, + "grad_norm": 28.879009246826172, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7078598737716675, + "logits/rejected": -0.6796781420707703, + "logps/chosen": -78.98861694335938, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -168.27285766601562, + "loss": 0.9551, + "margin_dpo/margin_mean": 26.070392608642578, + "margin_dpo/margin_std": 32.72665023803711, + "step": 81 + }, + { + "KL/chosen_KL_mean": -14.891376495361328, + "KL/mean": -25.239898681640625, + "KL/rejected_KL_mean": -35.58842086791992, + "KL/std": 25.123170852661133, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.026201102882623672, + "fcm_dpo/delta": -0.1510966718196869, + "fcm_dpo/margin": 20.697046279907227, + "fcm_dpo/q_t": 0.387561559677124, + "grad_norm": 31.851064682006836, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6644119024276733, + "logits/rejected": -0.6544766426086426, + "logps/chosen": -67.12376403808594, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -126.33167266845703, + "loss": 1.0941, + "margin_dpo/margin_mean": 20.697046279907227, + "margin_dpo/margin_std": 34.72496032714844, + "step": 82 + }, + { + "KL/chosen_KL_mean": -15.211023330688477, + "KL/mean": -27.510498046875, + "KL/rejected_KL_mean": -39.80997848510742, + "KL/std": 26.099361419677734, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.024797312915325165, + "fcm_dpo/delta": -0.22463998198509216, + "fcm_dpo/margin": 24.598949432373047, + "fcm_dpo/q_t": 0.36880677938461304, + "grad_norm": 28.356998443603516, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6986846327781677, + "logits/rejected": -0.6668576002120972, + "logps/chosen": -71.03840637207031, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -143.52587890625, + "loss": 0.9938, + "margin_dpo/margin_mean": 24.598949432373047, + "margin_dpo/margin_std": 31.020763397216797, + "step": 83 + }, + { + "KL/chosen_KL_mean": -14.247339248657227, + "KL/mean": -24.59140396118164, + "KL/rejected_KL_mean": -34.93547058105469, + "KL/std": 22.19933319091797, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.024027951061725616, + "fcm_dpo/delta": -0.10424363613128662, + "fcm_dpo/margin": 20.688127517700195, + "fcm_dpo/q_t": 0.3849903345108032, + "grad_norm": 25.717411041259766, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6441089510917664, + "logits/rejected": -0.6038364768028259, + "logps/chosen": -81.42350769042969, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -122.23406982421875, + "loss": 1.0218, + "margin_dpo/margin_mean": 20.688127517700195, + "margin_dpo/margin_std": 22.704191207885742, + "step": 84 + }, + { + "KL/chosen_KL_mean": -14.512044906616211, + "KL/mean": -25.26457977294922, + "KL/rejected_KL_mean": -36.01711654663086, + "KL/std": 22.689453125, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.02384367026388645, + "fcm_dpo/delta": -0.11910513043403625, + "fcm_dpo/margin": 21.50507354736328, + "fcm_dpo/q_t": 0.38629499077796936, + "grad_norm": 25.854524612426758, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6148737668991089, + "logits/rejected": -0.5798694491386414, + "logps/chosen": -72.91866302490234, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -114.65591430664062, + "loss": 1.038, + "margin_dpo/margin_mean": 21.50507354736328, + "margin_dpo/margin_std": 27.159027099609375, + "step": 85 + }, + { + "KL/chosen_KL_mean": -18.799100875854492, + "KL/mean": -30.1423397064209, + "KL/rejected_KL_mean": -41.485572814941406, + "KL/std": 30.83677864074707, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.02316327765583992, + "fcm_dpo/delta": -0.13226425647735596, + "fcm_dpo/margin": 22.686473846435547, + "fcm_dpo/q_t": 0.3980068266391754, + "grad_norm": 30.307323455810547, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.6908327341079712, + "logits/rejected": -0.6796331405639648, + "logps/chosen": -74.93656921386719, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -129.6072235107422, + "loss": 1.1226, + "margin_dpo/margin_mean": 22.686473846435547, + "margin_dpo/margin_std": 42.647315979003906, + "step": 86 + }, + { + "KL/chosen_KL_mean": -18.37569236755371, + "KL/mean": -31.817665100097656, + "KL/rejected_KL_mean": -45.259647369384766, + "KL/std": 30.71811294555664, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.022089457139372826, + "fcm_dpo/delta": -0.20784582197666168, + "fcm_dpo/margin": 26.883956909179688, + "fcm_dpo/q_t": 0.3717167377471924, + "grad_norm": 25.859588623046875, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7176273465156555, + "logits/rejected": -0.694380521774292, + "logps/chosen": -74.01178741455078, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -140.72723388671875, + "loss": 1.0284, + "margin_dpo/margin_mean": 26.883956909179688, + "margin_dpo/margin_std": 37.455299377441406, + "step": 87 + }, + { + "KL/chosen_KL_mean": -23.22958755493164, + "KL/mean": -33.348724365234375, + "KL/rejected_KL_mean": -43.467857360839844, + "KL/std": 32.11151123046875, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.021713685244321823, + "fcm_dpo/delta": -0.04214869812130928, + "fcm_dpo/margin": 20.238265991210938, + "fcm_dpo/q_t": 0.40647682547569275, + "grad_norm": 26.722354888916016, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.6343653202056885, + "logits/rejected": -0.6282986402511597, + "logps/chosen": -96.90074157714844, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -150.17636108398438, + "loss": 1.1297, + "margin_dpo/margin_mean": 20.23826789855957, + "margin_dpo/margin_std": 33.89949035644531, + "step": 88 + }, + { + "KL/chosen_KL_mean": -14.222597122192383, + "KL/mean": -25.512130737304688, + "KL/rejected_KL_mean": -36.801666259765625, + "KL/std": 26.90988540649414, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.02158905193209648, + "fcm_dpo/delta": -0.09193488210439682, + "fcm_dpo/margin": 22.579065322875977, + "fcm_dpo/q_t": 0.39170193672180176, + "grad_norm": 24.13078498840332, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7246202230453491, + "logits/rejected": -0.6930450797080994, + "logps/chosen": -74.8475112915039, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -118.88521575927734, + "loss": 1.0521, + "margin_dpo/margin_mean": 22.57906723022461, + "margin_dpo/margin_std": 30.714065551757812, + "step": 89 + }, + { + "KL/chosen_KL_mean": -18.214801788330078, + "KL/mean": -33.74324035644531, + "KL/rejected_KL_mean": -49.27167892456055, + "KL/std": 36.79106903076172, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.020788639783859253, + "fcm_dpo/delta": -0.2620236277580261, + "fcm_dpo/margin": 31.0568790435791, + "fcm_dpo/q_t": 0.3724457025527954, + "grad_norm": 25.449304580688477, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6592608690261841, + "logits/rejected": -0.6732791662216187, + "logps/chosen": -71.50011444091797, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -160.81639099121094, + "loss": 1.0294, + "margin_dpo/margin_mean": 31.056875228881836, + "margin_dpo/margin_std": 45.77299499511719, + "step": 90 + }, + { + "KL/chosen_KL_mean": -19.033781051635742, + "KL/mean": -31.643463134765625, + "KL/rejected_KL_mean": -44.253143310546875, + "KL/std": 29.290977478027344, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.020042097195982933, + "fcm_dpo/delta": -0.11093145608901978, + "fcm_dpo/margin": 25.21936798095703, + "fcm_dpo/q_t": 0.39225316047668457, + "grad_norm": 24.552165985107422, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.68781578540802, + "logits/rejected": -0.6556513905525208, + "logps/chosen": -80.83673858642578, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -132.12710571289062, + "loss": 1.0715, + "margin_dpo/margin_mean": 25.21936798095703, + "margin_dpo/margin_std": 38.131195068359375, + "step": 91 + }, + { + "KL/chosen_KL_mean": -17.03313636779785, + "KL/mean": -29.823741912841797, + "KL/rejected_KL_mean": -42.61435317993164, + "KL/std": 29.97732162475586, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.019645534455776215, + "fcm_dpo/delta": -0.1079995185136795, + "fcm_dpo/margin": 25.581214904785156, + "fcm_dpo/q_t": 0.3911270797252655, + "grad_norm": 22.701358795166016, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.7216691970825195, + "logits/rejected": -0.7003387212753296, + "logps/chosen": -68.67390441894531, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -120.49553680419922, + "loss": 1.0594, + "margin_dpo/margin_mean": 25.581214904785156, + "margin_dpo/margin_std": 36.2845458984375, + "step": 92 + }, + { + "KL/chosen_KL_mean": -19.190570831298828, + "KL/mean": -32.06718063354492, + "KL/rejected_KL_mean": -44.943782806396484, + "KL/std": 28.393882751464844, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.01917531155049801, + "fcm_dpo/delta": -0.0986010730266571, + "fcm_dpo/margin": 25.753215789794922, + "fcm_dpo/q_t": 0.39045190811157227, + "grad_norm": 23.477237701416016, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.7271374464035034, + "logits/rejected": -0.6787741780281067, + "logps/chosen": -71.71981048583984, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -122.10453033447266, + "loss": 1.0375, + "margin_dpo/margin_mean": 25.753215789794922, + "margin_dpo/margin_std": 32.94078826904297, + "step": 93 + }, + { + "KL/chosen_KL_mean": -20.50832176208496, + "KL/mean": -35.40428924560547, + "KL/rejected_KL_mean": -50.30025863647461, + "KL/std": 32.62531280517578, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.01849423348903656, + "fcm_dpo/delta": -0.16136708855628967, + "fcm_dpo/margin": 29.79193878173828, + "fcm_dpo/q_t": 0.37703630328178406, + "grad_norm": 23.016225814819336, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6730239391326904, + "logits/rejected": -0.659609317779541, + "logps/chosen": -81.7309341430664, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -149.8992919921875, + "loss": 0.9993, + "margin_dpo/margin_mean": 29.79193878173828, + "margin_dpo/margin_std": 34.16309356689453, + "step": 94 + }, + { + "KL/chosen_KL_mean": -19.582616806030273, + "KL/mean": -31.593730926513672, + "KL/rejected_KL_mean": -43.60485076904297, + "KL/std": 31.462764739990234, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.0182651337236166, + "fcm_dpo/delta": -0.04099061340093613, + "fcm_dpo/margin": 24.022232055664062, + "fcm_dpo/q_t": 0.3996763825416565, + "grad_norm": 21.716581344604492, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.6940039396286011, + "logits/rejected": -0.6668905019760132, + "logps/chosen": -72.10626220703125, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -119.48519897460938, + "loss": 1.0792, + "margin_dpo/margin_mean": 24.022232055664062, + "margin_dpo/margin_std": 32.96451187133789, + "step": 95 + }, + { + "KL/chosen_KL_mean": -19.07758140563965, + "KL/mean": -35.32679748535156, + "KL/rejected_KL_mean": -51.576019287109375, + "KL/std": 35.23541259765625, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.017663419246673584, + "fcm_dpo/delta": -0.1871272176504135, + "fcm_dpo/margin": 32.498435974121094, + "fcm_dpo/q_t": 0.3747464418411255, + "grad_norm": 23.28016471862793, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7074838876724243, + "logits/rejected": -0.6803252696990967, + "logps/chosen": -81.23455810546875, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -148.17202758789062, + "loss": 0.9924, + "margin_dpo/margin_mean": 32.498435974121094, + "margin_dpo/margin_std": 38.96044921875, + "step": 96 + }, + { + "KL/chosen_KL_mean": -20.322708129882812, + "KL/mean": -32.854896545410156, + "KL/rejected_KL_mean": -45.3870964050293, + "KL/std": 29.43494415283203, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.01744404248893261, + "fcm_dpo/delta": -0.10280635207891464, + "fcm_dpo/margin": 25.06438636779785, + "fcm_dpo/q_t": 0.39825475215911865, + "grad_norm": 22.761947631835938, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6467159390449524, + "logits/rejected": -0.605725884437561, + "logps/chosen": -74.96907043457031, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -122.35185241699219, + "loss": 1.0633, + "margin_dpo/margin_mean": 25.06438636779785, + "margin_dpo/margin_std": 29.242412567138672, + "step": 97 + }, + { + "KL/chosen_KL_mean": -25.34811019897461, + "KL/mean": -39.627140045166016, + "KL/rejected_KL_mean": -53.90617370605469, + "KL/std": 33.58491897583008, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.017004355788230896, + "fcm_dpo/delta": -0.09020677208900452, + "fcm_dpo/margin": 28.55807113647461, + "fcm_dpo/q_t": 0.389046847820282, + "grad_norm": 23.994243621826172, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6899088025093079, + "logits/rejected": -0.6754022240638733, + "logps/chosen": -90.60673522949219, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -150.43365478515625, + "loss": 1.0681, + "margin_dpo/margin_mean": 28.55807113647461, + "margin_dpo/margin_std": 40.3538818359375, + "step": 98 + }, + { + "KL/chosen_KL_mean": -19.761411666870117, + "KL/mean": -37.44667053222656, + "KL/rejected_KL_mean": -55.131927490234375, + "KL/std": 36.37155532836914, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.016426879912614822, + "fcm_dpo/delta": -0.1929803490638733, + "fcm_dpo/margin": 35.37051773071289, + "fcm_dpo/q_t": 0.37590575218200684, + "grad_norm": 20.758333206176758, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.6553323268890381, + "logits/rejected": -0.6570772528648376, + "logps/chosen": -65.39989471435547, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -141.56985473632812, + "loss": 0.9993, + "margin_dpo/margin_mean": 35.37051773071289, + "margin_dpo/margin_std": 45.35175323486328, + "step": 99 + }, + { + "KL/chosen_KL_mean": -22.288217544555664, + "KL/mean": -32.40540313720703, + "KL/rejected_KL_mean": -42.5225830078125, + "KL/std": 29.269241333007812, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.016305092722177505, + "fcm_dpo/delta": -0.005499512422829866, + "fcm_dpo/margin": 20.234359741210938, + "fcm_dpo/q_t": 0.424974650144577, + "grad_norm": 22.827594757080078, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6590077877044678, + "logits/rejected": -0.6173446178436279, + "logps/chosen": -79.8822021484375, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -116.58279418945312, + "loss": 1.1766, + "margin_dpo/margin_mean": 20.23436164855957, + "margin_dpo/margin_std": 37.6768684387207, + "step": 100 + }, + { + "KL/chosen_KL_mean": -27.39498519897461, + "KL/mean": -39.849342346191406, + "KL/rejected_KL_mean": -52.30369567871094, + "KL/std": 36.14341735839844, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.016109300777316093, + "fcm_dpo/delta": -0.057653024792671204, + "fcm_dpo/margin": 24.908700942993164, + "fcm_dpo/q_t": 0.4135899543762207, + "grad_norm": 22.6678524017334, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6499946117401123, + "logits/rejected": -0.6187979578971863, + "logps/chosen": -89.04383850097656, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -135.49337768554688, + "loss": 1.137, + "margin_dpo/margin_mean": 24.908700942993164, + "margin_dpo/margin_std": 43.55934143066406, + "step": 101 + }, + { + "KL/chosen_KL_mean": -29.505491256713867, + "KL/mean": -38.61754608154297, + "KL/rejected_KL_mean": -47.7296028137207, + "KL/std": 33.735816955566406, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.01608443260192871, + "fcm_dpo/delta": -0.015460515394806862, + "fcm_dpo/margin": 18.22411346435547, + "fcm_dpo/q_t": 0.4344051778316498, + "grad_norm": 25.596973419189453, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.6670002937316895, + "logits/rejected": -0.6216702461242676, + "logps/chosen": -93.58436584472656, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -115.91667938232422, + "loss": 1.2157, + "margin_dpo/margin_mean": 18.22411346435547, + "margin_dpo/margin_std": 40.039466857910156, + "step": 102 + }, + { + "KL/chosen_KL_mean": -23.598529815673828, + "KL/mean": -41.917877197265625, + "KL/rejected_KL_mean": -60.23723220825195, + "KL/std": 37.62096405029297, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.015599487349390984, + "fcm_dpo/delta": -0.18239150941371918, + "fcm_dpo/margin": 36.638694763183594, + "fcm_dpo/q_t": 0.3756001889705658, + "grad_norm": 22.023544311523438, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6658980846405029, + "logits/rejected": -0.6324506998062134, + "logps/chosen": -84.89781188964844, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -153.8099365234375, + "loss": 0.9995, + "margin_dpo/margin_mean": 36.63869094848633, + "margin_dpo/margin_std": 45.54207992553711, + "step": 103 + }, + { + "KL/chosen_KL_mean": -25.636720657348633, + "KL/mean": -43.03205490112305, + "KL/rejected_KL_mean": -60.42738342285156, + "KL/std": 41.528724670410156, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.015200499445199966, + "fcm_dpo/delta": -0.1358603835105896, + "fcm_dpo/margin": 34.790672302246094, + "fcm_dpo/q_t": 0.38911622762680054, + "grad_norm": 21.504518508911133, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.6759339570999146, + "logits/rejected": -0.6459177732467651, + "logps/chosen": -80.00949096679688, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -149.99209594726562, + "loss": 1.0545, + "margin_dpo/margin_mean": 34.790672302246094, + "margin_dpo/margin_std": 50.943355560302734, + "step": 104 + }, + { + "KL/chosen_KL_mean": -24.136260986328125, + "KL/mean": -48.879425048828125, + "KL/rejected_KL_mean": -73.62258911132812, + "KL/std": 40.33405303955078, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.014356976374983788, + "fcm_dpo/delta": -0.3339359164237976, + "fcm_dpo/margin": 49.486331939697266, + "fcm_dpo/q_t": 0.33943241834640503, + "grad_norm": 21.983217239379883, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.6709651350975037, + "logits/rejected": -0.6497222185134888, + "logps/chosen": -78.77520751953125, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -171.59609985351562, + "loss": 0.8859, + "margin_dpo/margin_mean": 49.486331939697266, + "margin_dpo/margin_std": 44.736106872558594, + "step": 105 + }, + { + "KL/chosen_KL_mean": -27.091875076293945, + "KL/mean": -42.523277282714844, + "KL/rejected_KL_mean": -57.954689025878906, + "KL/std": 35.19036102294922, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.013998525217175484, + "fcm_dpo/delta": -0.0336189940571785, + "fcm_dpo/margin": 30.862808227539062, + "fcm_dpo/q_t": 0.400464802980423, + "grad_norm": 20.296714782714844, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.6319311857223511, + "logits/rejected": -0.6033502817153931, + "logps/chosen": -81.92477416992188, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -143.17930603027344, + "loss": 1.0681, + "margin_dpo/margin_mean": 30.862808227539062, + "margin_dpo/margin_std": 38.33363342285156, + "step": 106 + }, + { + "KL/chosen_KL_mean": -32.350772857666016, + "KL/mean": -49.80072021484375, + "KL/rejected_KL_mean": -67.25065612792969, + "KL/std": 43.11651611328125, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.013766054064035416, + "fcm_dpo/delta": -0.08522772789001465, + "fcm_dpo/margin": 34.89988708496094, + "fcm_dpo/q_t": 0.3910776376724243, + "grad_norm": 20.588909149169922, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.6833256483078003, + "logits/rejected": -0.6406859159469604, + "logps/chosen": -102.05858612060547, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -161.99017333984375, + "loss": 1.0546, + "margin_dpo/margin_mean": 34.89988708496094, + "margin_dpo/margin_std": 45.648033142089844, + "step": 107 + }, + { + "KL/chosen_KL_mean": -28.926637649536133, + "KL/mean": -50.116329193115234, + "KL/rejected_KL_mean": -71.30602264404297, + "KL/std": 47.83903503417969, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.01340983621776104, + "fcm_dpo/delta": -0.17838259041309357, + "fcm_dpo/margin": 42.37938690185547, + "fcm_dpo/q_t": 0.38217076659202576, + "grad_norm": 20.71995735168457, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.6680444478988647, + "logits/rejected": -0.6493890285491943, + "logps/chosen": -84.9365234375, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -167.10203552246094, + "loss": 1.0426, + "margin_dpo/margin_mean": 42.37937927246094, + "margin_dpo/margin_std": 61.89811706542969, + "step": 108 + }, + { + "KL/chosen_KL_mean": -27.01991844177246, + "KL/mean": -49.54051971435547, + "KL/rejected_KL_mean": -72.06112670898438, + "KL/std": 45.74040985107422, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.012864284217357635, + "fcm_dpo/delta": -0.1914011538028717, + "fcm_dpo/margin": 45.04120635986328, + "fcm_dpo/q_t": 0.37112706899642944, + "grad_norm": 21.676660537719727, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.5962870121002197, + "logits/rejected": -0.575408935546875, + "logps/chosen": -89.90541076660156, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -170.74685668945312, + "loss": 0.9974, + "margin_dpo/margin_mean": 45.04120635986328, + "margin_dpo/margin_std": 54.460533142089844, + "step": 109 + }, + { + "KL/chosen_KL_mean": -28.106735229492188, + "KL/mean": -48.37699890136719, + "KL/rejected_KL_mean": -68.64726257324219, + "KL/std": 45.66210174560547, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.012337589636445045, + "fcm_dpo/delta": -0.17745205760002136, + "fcm_dpo/margin": 40.54052734375, + "fcm_dpo/q_t": 0.38920724391937256, + "grad_norm": 18.69563865661621, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.5969946384429932, + "logits/rejected": -0.5616341233253479, + "logps/chosen": -86.86042022705078, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -148.39727783203125, + "loss": 1.0598, + "margin_dpo/margin_mean": 40.54052734375, + "margin_dpo/margin_std": 55.06962585449219, + "step": 110 + }, + { + "KL/chosen_KL_mean": -31.72068214416504, + "KL/mean": -54.84687042236328, + "KL/rejected_KL_mean": -77.97306060791016, + "KL/std": 51.633670806884766, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.011975137516856194, + "fcm_dpo/delta": -0.16438831388950348, + "fcm_dpo/margin": 46.252384185791016, + "fcm_dpo/q_t": 0.37904852628707886, + "grad_norm": 21.229190826416016, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.6474937200546265, + "logits/rejected": -0.626065731048584, + "logps/chosen": -100.34478759765625, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -176.40191650390625, + "loss": 1.0394, + "margin_dpo/margin_mean": 46.25238800048828, + "margin_dpo/margin_std": 63.884552001953125, + "step": 111 + }, + { + "KL/chosen_KL_mean": -29.243896484375, + "KL/mean": -43.83025360107422, + "KL/rejected_KL_mean": -58.4166145324707, + "KL/std": 35.578922271728516, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.011899566277861595, + "fcm_dpo/delta": -0.016355372965335846, + "fcm_dpo/margin": 29.17272186279297, + "fcm_dpo/q_t": 0.4197250008583069, + "grad_norm": 19.126829147338867, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.6047525405883789, + "logits/rejected": -0.565899133682251, + "logps/chosen": -79.49353790283203, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -123.1910400390625, + "loss": 1.1332, + "margin_dpo/margin_mean": 29.17272186279297, + "margin_dpo/margin_std": 43.3043212890625, + "step": 112 + }, + { + "KL/chosen_KL_mean": -36.891048431396484, + "KL/mean": -52.48798751831055, + "KL/rejected_KL_mean": -68.08493041992188, + "KL/std": 37.52532196044922, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.011951886117458344, + "fcm_dpo/delta": 0.028221070766448975, + "fcm_dpo/margin": 31.19386863708496, + "fcm_dpo/q_t": 0.4139564633369446, + "grad_norm": 19.227039337158203, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.5778098106384277, + "logits/rejected": -0.5337458848953247, + "logps/chosen": -103.60400390625, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -146.0536346435547, + "loss": 1.1059, + "margin_dpo/margin_mean": 31.193866729736328, + "margin_dpo/margin_std": 41.64220428466797, + "step": 113 + }, + { + "KL/chosen_KL_mean": -33.76496887207031, + "KL/mean": -58.71270751953125, + "KL/rejected_KL_mean": -83.66044616699219, + "KL/std": 55.96549987792969, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.011562490835785866, + "fcm_dpo/delta": -0.18971961736679077, + "fcm_dpo/margin": 49.89548110961914, + "fcm_dpo/q_t": 0.3786962628364563, + "grad_norm": 21.20136833190918, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.5907497406005859, + "logits/rejected": -0.5589549541473389, + "logps/chosen": -91.550048828125, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -170.77011108398438, + "loss": 1.0113, + "margin_dpo/margin_mean": 49.89548110961914, + "margin_dpo/margin_std": 67.87698364257812, + "step": 114 + }, + { + "KL/chosen_KL_mean": -43.85015869140625, + "KL/mean": -60.76088333129883, + "KL/rejected_KL_mean": -77.67160034179688, + "KL/std": 53.124351501464844, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.011494990438222885, + "fcm_dpo/delta": -0.022934848442673683, + "fcm_dpo/margin": 33.821441650390625, + "fcm_dpo/q_t": 0.41787421703338623, + "grad_norm": 25.603805541992188, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.6201134920120239, + "logits/rejected": -0.6015622615814209, + "logps/chosen": -109.43280029296875, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -176.23712158203125, + "loss": 1.176, + "margin_dpo/margin_mean": 33.821441650390625, + "margin_dpo/margin_std": 70.07882690429688, + "step": 115 + }, + { + "KL/chosen_KL_mean": -32.55548095703125, + "KL/mean": -51.60619354248047, + "KL/rejected_KL_mean": -70.65690612792969, + "KL/std": 44.91873550415039, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.011454465799033642, + "fcm_dpo/delta": -0.038328371942043304, + "fcm_dpo/margin": 38.101409912109375, + "fcm_dpo/q_t": 0.40130701661109924, + "grad_norm": 21.188064575195312, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.6170135736465454, + "logits/rejected": -0.6094729900360107, + "logps/chosen": -83.95579528808594, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -151.1787567138672, + "loss": 1.0687, + "margin_dpo/margin_mean": 38.101409912109375, + "margin_dpo/margin_std": 48.773841857910156, + "step": 116 + }, + { + "KL/chosen_KL_mean": -42.18211364746094, + "KL/mean": -57.61778259277344, + "KL/rejected_KL_mean": -73.05345153808594, + "KL/std": 47.762115478515625, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.011464458890259266, + "fcm_dpo/delta": 0.04775825887918472, + "fcm_dpo/margin": 30.871334075927734, + "fcm_dpo/q_t": 0.42048314213752747, + "grad_norm": 26.351938247680664, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.5902009010314941, + "logits/rejected": -0.5555441975593567, + "logps/chosen": -111.48052215576172, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -139.63743591308594, + "loss": 1.1519, + "margin_dpo/margin_mean": 30.871334075927734, + "margin_dpo/margin_std": 54.229087829589844, + "step": 117 + }, + { + "KL/chosen_KL_mean": -33.18445587158203, + "KL/mean": -52.039398193359375, + "KL/rejected_KL_mean": -70.89434051513672, + "KL/std": 42.98170471191406, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.011441653594374657, + "fcm_dpo/delta": -0.03286949172616005, + "fcm_dpo/margin": 37.70988464355469, + "fcm_dpo/q_t": 0.4025983214378357, + "grad_norm": 20.147180557250977, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.6222990155220032, + "logits/rejected": -0.6042829751968384, + "logps/chosen": -88.825439453125, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -146.5634002685547, + "loss": 1.0729, + "margin_dpo/margin_mean": 37.70988464355469, + "margin_dpo/margin_std": 49.364845275878906, + "step": 118 + }, + { + "KL/chosen_KL_mean": -45.622764587402344, + "KL/mean": -64.33576202392578, + "KL/rejected_KL_mean": -83.04876708984375, + "KL/std": 48.650794982910156, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.011235379613935947, + "fcm_dpo/delta": -0.07929864525794983, + "fcm_dpo/margin": 37.426002502441406, + "fcm_dpo/q_t": 0.40809717774391174, + "grad_norm": 22.4027042388916, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.6055055856704712, + "logits/rejected": -0.5936990976333618, + "logps/chosen": -119.13296508789062, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -186.02606201171875, + "loss": 1.1117, + "margin_dpo/margin_mean": 37.426002502441406, + "margin_dpo/margin_std": 59.61151885986328, + "step": 119 + }, + { + "KL/chosen_KL_mean": -46.729209899902344, + "KL/mean": -72.6898193359375, + "KL/rejected_KL_mean": -98.65043640136719, + "KL/std": 60.77156066894531, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.010935020633041859, + "fcm_dpo/delta": -0.17799094319343567, + "fcm_dpo/margin": 51.921226501464844, + "fcm_dpo/q_t": 0.3748236298561096, + "grad_norm": 21.275100708007812, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.6155554056167603, + "logits/rejected": -0.5853316783905029, + "logps/chosen": -123.51004028320312, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -206.67416381835938, + "loss": 1.0026, + "margin_dpo/margin_mean": 51.92122268676758, + "margin_dpo/margin_std": 63.717559814453125, + "step": 120 + }, + { + "KL/chosen_KL_mean": -45.204002380371094, + "KL/mean": -73.95439147949219, + "KL/rejected_KL_mean": -102.70478820800781, + "KL/std": 59.39262008666992, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.010449431836605072, + "fcm_dpo/delta": -0.21527445316314697, + "fcm_dpo/margin": 57.50080108642578, + "fcm_dpo/q_t": 0.37026768922805786, + "grad_norm": 23.518917083740234, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.5353620052337646, + "logits/rejected": -0.5309556722640991, + "logps/chosen": -106.993896484375, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -212.69935607910156, + "loss": 1.0032, + "margin_dpo/margin_mean": 57.50080108642578, + "margin_dpo/margin_std": 74.40509033203125, + "step": 121 + }, + { + "KL/chosen_KL_mean": -41.248985290527344, + "KL/mean": -76.55476379394531, + "KL/rejected_KL_mean": -111.86054992675781, + "KL/std": 67.27115631103516, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.009908626787364483, + "fcm_dpo/delta": -0.322030246257782, + "fcm_dpo/margin": 70.61155700683594, + "fcm_dpo/q_t": 0.3457896113395691, + "grad_norm": 22.932233810424805, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.530424177646637, + "logits/rejected": -0.5468406081199646, + "logps/chosen": -88.15119934082031, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -218.57473754882812, + "loss": 0.9079, + "margin_dpo/margin_mean": 70.61155700683594, + "margin_dpo/margin_std": 70.593505859375, + "step": 122 + }, + { + "KL/chosen_KL_mean": -45.29631042480469, + "KL/mean": -68.32212829589844, + "KL/rejected_KL_mean": -91.34793090820312, + "KL/std": 57.074432373046875, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.009618120267987251, + "fcm_dpo/delta": -0.04580308869481087, + "fcm_dpo/margin": 46.05162048339844, + "fcm_dpo/q_t": 0.40427830815315247, + "grad_norm": 20.790943145751953, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.5321049094200134, + "logits/rejected": -0.5150766372680664, + "logps/chosen": -106.63494873046875, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -179.12332153320312, + "loss": 1.115, + "margin_dpo/margin_mean": 46.05162048339844, + "margin_dpo/margin_std": 76.61585998535156, + "step": 123 + }, + { + "KL/chosen_KL_mean": -52.424560546875, + "KL/mean": -86.89698791503906, + "KL/rejected_KL_mean": -121.36941528320312, + "KL/std": 79.08882141113281, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.009263862855732441, + "fcm_dpo/delta": -0.25586703419685364, + "fcm_dpo/margin": 68.94483947753906, + "fcm_dpo/q_t": 0.3710143566131592, + "grad_norm": 25.040687561035156, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.5144941806793213, + "logits/rejected": -0.5171750783920288, + "logps/chosen": -123.87289428710938, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -238.94998168945312, + "loss": 1.0207, + "margin_dpo/margin_mean": 68.94483947753906, + "margin_dpo/margin_std": 99.44221496582031, + "step": 124 + }, + { + "KL/chosen_KL_mean": -43.17867660522461, + "KL/mean": -73.51754760742188, + "KL/rejected_KL_mean": -103.85641479492188, + "KL/std": 66.70317840576172, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.008978080004453659, + "fcm_dpo/delta": -0.1529029756784439, + "fcm_dpo/margin": 60.67774963378906, + "fcm_dpo/q_t": 0.38298797607421875, + "grad_norm": 19.180618286132812, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.49021369218826294, + "logits/rejected": -0.4929278790950775, + "logps/chosen": -93.31561279296875, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -187.84503173828125, + "loss": 1.037, + "margin_dpo/margin_mean": 60.67774963378906, + "margin_dpo/margin_std": 83.27035522460938, + "step": 125 + }, + { + "KL/chosen_KL_mean": -46.657310485839844, + "KL/mean": -75.00636291503906, + "KL/rejected_KL_mean": -103.35540771484375, + "KL/std": 60.334869384765625, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.008742941543459892, + "fcm_dpo/delta": -0.1009548008441925, + "fcm_dpo/margin": 56.6981086730957, + "fcm_dpo/q_t": 0.39125490188598633, + "grad_norm": 20.15732765197754, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.538190484046936, + "logits/rejected": -0.5352437496185303, + "logps/chosen": -102.32437896728516, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -201.48516845703125, + "loss": 1.0508, + "margin_dpo/margin_mean": 56.69811248779297, + "margin_dpo/margin_std": 76.94652557373047, + "step": 126 + }, + { + "KL/chosen_KL_mean": -44.97933578491211, + "KL/mean": -65.41487121582031, + "KL/rejected_KL_mean": -85.85040283203125, + "KL/std": 52.38228988647461, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.008773598819971085, + "fcm_dpo/delta": 0.04293098300695419, + "fcm_dpo/margin": 40.87106704711914, + "fcm_dpo/q_t": 0.4170472025871277, + "grad_norm": 20.807865142822266, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.5098677277565002, + "logits/rejected": -0.49067068099975586, + "logps/chosen": -101.53401184082031, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -162.64617919921875, + "loss": 1.1387, + "margin_dpo/margin_mean": 40.871063232421875, + "margin_dpo/margin_std": 66.03770446777344, + "step": 127 + }, + { + "KL/chosen_KL_mean": -47.79435729980469, + "KL/mean": -69.45096588134766, + "KL/rejected_KL_mean": -91.10757446289062, + "KL/std": 61.347110748291016, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.008761574514210224, + "fcm_dpo/delta": -0.031184187158942223, + "fcm_dpo/margin": 43.31322479248047, + "fcm_dpo/q_t": 0.4151967167854309, + "grad_norm": 27.38097381591797, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.5048182606697083, + "logits/rejected": -0.4856981039047241, + "logps/chosen": -105.91531372070312, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -167.54653930664062, + "loss": 1.1483, + "margin_dpo/margin_mean": 43.313228607177734, + "margin_dpo/margin_std": 76.85040283203125, + "step": 128 + }, + { + "KL/chosen_KL_mean": -57.734134674072266, + "KL/mean": -80.57646179199219, + "KL/rejected_KL_mean": -103.4188003540039, + "KL/std": 67.9546127319336, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.00874091312289238, + "fcm_dpo/delta": 0.0006930278614163399, + "fcm_dpo/margin": 45.684661865234375, + "fcm_dpo/q_t": 0.4161832928657532, + "grad_norm": 19.95856475830078, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.5131363868713379, + "logits/rejected": -0.5023560523986816, + "logps/chosen": -124.6505126953125, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -200.06100463867188, + "loss": 1.1548, + "margin_dpo/margin_mean": 45.68466567993164, + "margin_dpo/margin_std": 88.41867065429688, + "step": 129 + }, + { + "KL/chosen_KL_mean": -42.62668228149414, + "KL/mean": -76.86021423339844, + "KL/rejected_KL_mean": -111.09373474121094, + "KL/std": 69.08213806152344, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.008568612858653069, + "fcm_dpo/delta": -0.19837135076522827, + "fcm_dpo/margin": 68.46705627441406, + "fcm_dpo/q_t": 0.3735218346118927, + "grad_norm": 20.97186279296875, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.4922315776348114, + "logits/rejected": -0.48675650358200073, + "logps/chosen": -87.29353332519531, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -193.87539672851562, + "loss": 1.0054, + "margin_dpo/margin_mean": 68.46705627441406, + "margin_dpo/margin_std": 84.42669677734375, + "step": 130 + }, + { + "KL/chosen_KL_mean": -38.267921447753906, + "KL/mean": -72.79769897460938, + "KL/rejected_KL_mean": -107.32746887207031, + "KL/std": 69.21115112304688, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.008169912733137608, + "fcm_dpo/delta": -0.17431190609931946, + "fcm_dpo/margin": 69.05953979492188, + "fcm_dpo/q_t": 0.3714016377925873, + "grad_norm": 30.622962951660156, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.49250200390815735, + "logits/rejected": -0.4990604817867279, + "logps/chosen": -83.19251251220703, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -195.771484375, + "loss": 0.9865, + "margin_dpo/margin_mean": 69.05953979492188, + "margin_dpo/margin_std": 75.71206665039062, + "step": 131 + }, + { + "KL/chosen_KL_mean": -50.39973449707031, + "KL/mean": -78.53848266601562, + "KL/rejected_KL_mean": -106.67723083496094, + "KL/std": 68.39564514160156, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.008026429452002048, + "fcm_dpo/delta": -0.05434757471084595, + "fcm_dpo/margin": 56.277496337890625, + "fcm_dpo/q_t": 0.40332505106925964, + "grad_norm": 19.994808197021484, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.5088815689086914, + "logits/rejected": -0.5071790218353271, + "logps/chosen": -109.40081787109375, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -194.5693817138672, + "loss": 1.0936, + "margin_dpo/margin_mean": 56.277503967285156, + "margin_dpo/margin_std": 88.92678833007812, + "step": 132 + }, + { + "KL/chosen_KL_mean": -59.73477554321289, + "KL/mean": -84.5237045288086, + "KL/rejected_KL_mean": -109.3126220703125, + "KL/std": 61.86600875854492, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.008008632808923721, + "fcm_dpo/delta": 0.003002084791660309, + "fcm_dpo/margin": 49.57785415649414, + "fcm_dpo/q_t": 0.41264668107032776, + "grad_norm": 27.239116668701172, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.5110952854156494, + "logits/rejected": -0.4978069067001343, + "logps/chosen": -126.33927917480469, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -205.64617919921875, + "loss": 1.1171, + "margin_dpo/margin_mean": 49.577850341796875, + "margin_dpo/margin_std": 78.71560668945312, + "step": 133 + }, + { + "KL/chosen_KL_mean": -47.09953689575195, + "KL/mean": -73.70185852050781, + "KL/rejected_KL_mean": -100.30419158935547, + "KL/std": 59.980159759521484, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.008018873631954193, + "fcm_dpo/delta": -0.027966132387518883, + "fcm_dpo/margin": 53.20465087890625, + "fcm_dpo/q_t": 0.40397584438323975, + "grad_norm": 18.37006950378418, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.4879264831542969, + "logits/rejected": -0.4753343164920807, + "logps/chosen": -99.16879272460938, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -187.95870971679688, + "loss": 1.0734, + "margin_dpo/margin_mean": 53.20465087890625, + "margin_dpo/margin_std": 69.51371765136719, + "step": 134 + }, + { + "KL/chosen_KL_mean": -52.43537902832031, + "KL/mean": -90.9434814453125, + "KL/rejected_KL_mean": -129.4515838623047, + "KL/std": 80.5875244140625, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.007736685685813427, + "fcm_dpo/delta": -0.20829075574874878, + "fcm_dpo/margin": 77.01622009277344, + "fcm_dpo/q_t": 0.3724926710128784, + "grad_norm": 23.101228713989258, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.4772408604621887, + "logits/rejected": -0.5114985704421997, + "logps/chosen": -102.78923797607422, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -245.43133544921875, + "loss": 1.0, + "margin_dpo/margin_mean": 77.01622009277344, + "margin_dpo/margin_std": 97.571044921875, + "step": 135 + }, + { + "KL/chosen_KL_mean": -62.10560607910156, + "KL/mean": -87.8612060546875, + "KL/rejected_KL_mean": -113.61680603027344, + "KL/std": 73.29806518554688, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.007571512833237648, + "fcm_dpo/delta": -0.05470416694879532, + "fcm_dpo/margin": 51.511207580566406, + "fcm_dpo/q_t": 0.41969025135040283, + "grad_norm": 20.06012535095215, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.47930270433425903, + "logits/rejected": -0.4716784358024597, + "logps/chosen": -127.17811584472656, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -209.9380340576172, + "loss": 1.1461, + "margin_dpo/margin_mean": 51.511207580566406, + "margin_dpo/margin_std": 94.14688110351562, + "step": 136 + }, + { + "KL/chosen_KL_mean": -59.31104278564453, + "KL/mean": -99.50381469726562, + "KL/rejected_KL_mean": -139.69659423828125, + "KL/std": 96.50749206542969, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.007339527830481529, + "fcm_dpo/delta": -0.20216205716133118, + "fcm_dpo/margin": 80.38555145263672, + "fcm_dpo/q_t": 0.37922900915145874, + "grad_norm": 17.869787216186523, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.3955429792404175, + "logits/rejected": -0.42269080877304077, + "logps/chosen": -108.07015991210938, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -253.56036376953125, + "loss": 1.0311, + "margin_dpo/margin_mean": 80.38554382324219, + "margin_dpo/margin_std": 116.00267028808594, + "step": 137 + }, + { + "KL/chosen_KL_mean": -61.83498001098633, + "KL/mean": -92.16122436523438, + "KL/rejected_KL_mean": -122.48748779296875, + "KL/std": 73.12271881103516, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.007240916136652231, + "fcm_dpo/delta": -0.0409746915102005, + "fcm_dpo/margin": 60.652503967285156, + "fcm_dpo/q_t": 0.39921319484710693, + "grad_norm": 20.14948272705078, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.4438355565071106, + "logits/rejected": -0.43297797441482544, + "logps/chosen": -122.35462951660156, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -215.68443298339844, + "loss": 1.0626, + "margin_dpo/margin_mean": 60.652503967285156, + "margin_dpo/margin_std": 74.87147521972656, + "step": 138 + }, + { + "KL/chosen_KL_mean": -51.901573181152344, + "KL/mean": -88.16980743408203, + "KL/rejected_KL_mean": -124.43804931640625, + "KL/std": 69.6425552368164, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.007098979316651821, + "fcm_dpo/delta": -0.12100963294506073, + "fcm_dpo/margin": 72.53648376464844, + "fcm_dpo/q_t": 0.38553667068481445, + "grad_norm": 18.537273406982422, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.42982420325279236, + "logits/rejected": -0.4203348755836487, + "logps/chosen": -98.79295349121094, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -204.16603088378906, + "loss": 1.0258, + "margin_dpo/margin_mean": 72.53648376464844, + "margin_dpo/margin_std": 90.41792297363281, + "step": 139 + }, + { + "KL/chosen_KL_mean": -62.657005310058594, + "KL/mean": -95.34037780761719, + "KL/rejected_KL_mean": -128.02374267578125, + "KL/std": 77.21334838867188, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.006975092925131321, + "fcm_dpo/delta": -0.05864529311656952, + "fcm_dpo/margin": 65.36674499511719, + "fcm_dpo/q_t": 0.3983650803565979, + "grad_norm": 20.0074520111084, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.43783557415008545, + "logits/rejected": -0.4252389371395111, + "logps/chosen": -121.63172149658203, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -211.307861328125, + "loss": 1.0759, + "margin_dpo/margin_mean": 65.36674499511719, + "margin_dpo/margin_std": 92.9735107421875, + "step": 140 + }, + { + "KL/chosen_KL_mean": -69.94717407226562, + "KL/mean": -101.20220947265625, + "KL/rejected_KL_mean": -132.45721435546875, + "KL/std": 85.00312805175781, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.006927805487066507, + "fcm_dpo/delta": -0.03453919664025307, + "fcm_dpo/margin": 62.510040283203125, + "fcm_dpo/q_t": 0.40259939432144165, + "grad_norm": 24.273269653320312, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.4435374140739441, + "logits/rejected": -0.43256574869155884, + "logps/chosen": -145.02284240722656, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -230.64947509765625, + "loss": 1.1044, + "margin_dpo/margin_mean": 62.510040283203125, + "margin_dpo/margin_std": 98.28890991210938, + "step": 141 + }, + { + "KL/chosen_KL_mean": -71.40181732177734, + "KL/mean": -106.93650817871094, + "KL/rejected_KL_mean": -142.47119140625, + "KL/std": 92.28612518310547, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.006845604162663221, + "fcm_dpo/delta": -0.09091417491436005, + "fcm_dpo/margin": 71.06938934326172, + "fcm_dpo/q_t": 0.3944661617279053, + "grad_norm": 24.990633010864258, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.40035754442214966, + "logits/rejected": -0.39934098720550537, + "logps/chosen": -129.42974853515625, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -237.05343627929688, + "loss": 1.0859, + "margin_dpo/margin_mean": 71.06938934326172, + "margin_dpo/margin_std": 108.50109100341797, + "step": 142 + }, + { + "KL/chosen_KL_mean": -75.55914306640625, + "KL/mean": -98.31578063964844, + "KL/rejected_KL_mean": -121.07241821289062, + "KL/std": 81.29641723632812, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.006790860556066036, + "fcm_dpo/delta": 0.004169495776295662, + "fcm_dpo/margin": 45.513275146484375, + "fcm_dpo/q_t": 0.43299609422683716, + "grad_norm": 22.193300247192383, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.37825581431388855, + "logits/rejected": -0.3710605800151825, + "logps/chosen": -133.1555938720703, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -200.07199096679688, + "loss": 1.1974, + "margin_dpo/margin_mean": 45.513275146484375, + "margin_dpo/margin_std": 94.83728790283203, + "step": 143 + }, + { + "KL/chosen_KL_mean": -67.32421875, + "KL/mean": -95.0378189086914, + "KL/rejected_KL_mean": -122.75141906738281, + "KL/std": 68.96160888671875, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.006809500977396965, + "fcm_dpo/delta": 0.023459728807210922, + "fcm_dpo/margin": 55.42720031738281, + "fcm_dpo/q_t": 0.41421592235565186, + "grad_norm": 19.899524688720703, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.39314860105514526, + "logits/rejected": -0.38118699193000793, + "logps/chosen": -127.23058319091797, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -204.75167846679688, + "loss": 1.1157, + "margin_dpo/margin_mean": 55.42720031738281, + "margin_dpo/margin_std": 81.08219909667969, + "step": 144 + }, + { + "KL/chosen_KL_mean": -65.09317016601562, + "KL/mean": -95.10823822021484, + "KL/rejected_KL_mean": -125.1233139038086, + "KL/std": 69.72821807861328, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.0068089039996266365, + "fcm_dpo/delta": -0.009127567522227764, + "fcm_dpo/margin": 60.030147552490234, + "fcm_dpo/q_t": 0.4070216715335846, + "grad_norm": 23.450458526611328, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.427666574716568, + "logits/rejected": -0.41289660334587097, + "logps/chosen": -121.69383239746094, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -202.9896240234375, + "loss": 1.0948, + "margin_dpo/margin_mean": 60.03014373779297, + "margin_dpo/margin_std": 84.30757141113281, + "step": 145 + }, + { + "KL/chosen_KL_mean": -88.69615936279297, + "KL/mean": -111.81280517578125, + "KL/rejected_KL_mean": -134.92945861816406, + "KL/std": 75.6089096069336, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.006827444303780794, + "fcm_dpo/delta": 0.012531173415482044, + "fcm_dpo/margin": 46.23329162597656, + "fcm_dpo/q_t": 0.4276990294456482, + "grad_norm": 26.07567596435547, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.39292311668395996, + "logits/rejected": -0.3666006624698639, + "logps/chosen": -154.6966094970703, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -216.63223266601562, + "loss": 1.188, + "margin_dpo/margin_mean": 46.23329162597656, + "margin_dpo/margin_std": 90.79597473144531, + "step": 146 + }, + { + "KL/chosen_KL_mean": -65.11907958984375, + "KL/mean": -96.63715362548828, + "KL/rejected_KL_mean": -128.1552276611328, + "KL/std": 76.17960357666016, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.00678680744022131, + "fcm_dpo/delta": -0.029158955439925194, + "fcm_dpo/margin": 63.03614807128906, + "fcm_dpo/q_t": 0.40489357709884644, + "grad_norm": 20.027523040771484, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.42425787448883057, + "logits/rejected": -0.40518224239349365, + "logps/chosen": -118.52456665039062, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -199.54583740234375, + "loss": 1.097, + "margin_dpo/margin_mean": 63.03614807128906, + "margin_dpo/margin_std": 94.40005493164062, + "step": 147 + }, + { + "KL/chosen_KL_mean": -63.721580505371094, + "KL/mean": -90.95237731933594, + "KL/rejected_KL_mean": -118.18316650390625, + "KL/std": 76.05411529541016, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.006736146751791239, + "fcm_dpo/delta": -0.07659874856472015, + "fcm_dpo/margin": 54.46158981323242, + "fcm_dpo/q_t": 0.4183466136455536, + "grad_norm": 18.79749870300293, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.47092750668525696, + "logits/rejected": -0.4641035199165344, + "logps/chosen": -128.65866088867188, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -221.27700805664062, + "loss": 1.129, + "margin_dpo/margin_mean": 54.46158981323242, + "margin_dpo/margin_std": 80.068359375, + "step": 148 + }, + { + "KL/chosen_KL_mean": -60.64297103881836, + "KL/mean": -94.22190856933594, + "KL/rejected_KL_mean": -127.80084228515625, + "KL/std": 68.38766479492188, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.006552124861627817, + "fcm_dpo/delta": -0.10004719346761703, + "fcm_dpo/margin": 67.15786743164062, + "fcm_dpo/q_t": 0.3984683156013489, + "grad_norm": 17.950054168701172, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.474928081035614, + "logits/rejected": -0.4603117108345032, + "logps/chosen": -119.11672973632812, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -227.11558532714844, + "loss": 1.0638, + "margin_dpo/margin_mean": 67.15786743164062, + "margin_dpo/margin_std": 80.81826782226562, + "step": 149 + }, + { + "KL/chosen_KL_mean": -53.816017150878906, + "KL/mean": -86.77810668945312, + "KL/rejected_KL_mean": -119.74018096923828, + "KL/std": 80.36477661132812, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.006446311715990305, + "fcm_dpo/delta": -0.08140614628791809, + "fcm_dpo/margin": 65.92416381835938, + "fcm_dpo/q_t": 0.40723806619644165, + "grad_norm": 17.54256248474121, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.4389897584915161, + "logits/rejected": -0.442792683839798, + "logps/chosen": -99.5218276977539, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -203.08778381347656, + "loss": 1.0893, + "margin_dpo/margin_mean": 65.92416381835938, + "margin_dpo/margin_std": 95.80305480957031, + "step": 150 + }, + { + "KL/chosen_KL_mean": -67.46832275390625, + "KL/mean": -100.74046325683594, + "KL/rejected_KL_mean": -134.01260375976562, + "KL/std": 76.1561050415039, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.0064178211614489555, + "fcm_dpo/delta": -0.02828397788107395, + "fcm_dpo/margin": 66.54426574707031, + "fcm_dpo/q_t": 0.4017104506492615, + "grad_norm": 19.113588333129883, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.5316263437271118, + "logits/rejected": -0.5064392685890198, + "logps/chosen": -138.03915405273438, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -234.47640991210938, + "loss": 1.0698, + "margin_dpo/margin_mean": 66.54427337646484, + "margin_dpo/margin_std": 83.10430908203125, + "step": 151 + }, + { + "KL/chosen_KL_mean": -59.05611038208008, + "KL/mean": -99.39572143554688, + "KL/rejected_KL_mean": -139.73533630371094, + "KL/std": 78.44137573242188, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.006315959617495537, + "fcm_dpo/delta": -0.11529232561588287, + "fcm_dpo/margin": 80.67922973632812, + "fcm_dpo/q_t": 0.3860214352607727, + "grad_norm": 20.485200881958008, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.4727810323238373, + "logits/rejected": -0.47146517038345337, + "logps/chosen": -119.22049713134766, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -245.87579345703125, + "loss": 1.0265, + "margin_dpo/margin_mean": 80.67922973632812, + "margin_dpo/margin_std": 96.18466186523438, + "step": 152 + }, + { + "KL/chosen_KL_mean": -60.14429473876953, + "KL/mean": -94.39277648925781, + "KL/rejected_KL_mean": -128.6412353515625, + "KL/std": 85.70268249511719, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.006248671095818281, + "fcm_dpo/delta": -0.02928725816309452, + "fcm_dpo/margin": 68.4969482421875, + "fcm_dpo/q_t": 0.4066329896450043, + "grad_norm": 14.968043327331543, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.4556788206100464, + "logits/rejected": -0.4505578875541687, + "logps/chosen": -116.4595718383789, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -214.29708862304688, + "loss": 1.0985, + "margin_dpo/margin_mean": 68.4969482421875, + "margin_dpo/margin_std": 105.18212890625, + "step": 153 + }, + { + "KL/chosen_KL_mean": -72.00877380371094, + "KL/mean": -104.7803955078125, + "KL/rejected_KL_mean": -137.55201721191406, + "KL/std": 86.04232788085938, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.006242698989808559, + "fcm_dpo/delta": -0.009834952652454376, + "fcm_dpo/margin": 65.54324340820312, + "fcm_dpo/q_t": 0.40968552231788635, + "grad_norm": 18.247264862060547, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.4711272120475769, + "logits/rejected": -0.4746021628379822, + "logps/chosen": -134.7513427734375, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -241.79620361328125, + "loss": 1.1281, + "margin_dpo/margin_mean": 65.54324340820312, + "margin_dpo/margin_std": 109.99638366699219, + "step": 154 + }, + { + "KL/chosen_KL_mean": -65.82139587402344, + "KL/mean": -99.0433349609375, + "KL/rejected_KL_mean": -132.2652587890625, + "KL/std": 79.94084167480469, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.006218398921191692, + "fcm_dpo/delta": -0.01384439505636692, + "fcm_dpo/margin": 66.44386291503906, + "fcm_dpo/q_t": 0.4070393145084381, + "grad_norm": 18.415481567382812, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.47266900539398193, + "logits/rejected": -0.45449507236480713, + "logps/chosen": -126.47457885742188, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -209.75746154785156, + "loss": 1.1023, + "margin_dpo/margin_mean": 66.44386291503906, + "margin_dpo/margin_std": 99.56658935546875, + "step": 155 + }, + { + "KL/chosen_KL_mean": -92.22872924804688, + "KL/mean": -111.12690734863281, + "KL/rejected_KL_mean": -130.0250701904297, + "KL/std": 85.2044677734375, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.006197832524776459, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 37.79635238647461, + "fcm_dpo/q_t": 0.44647374749183655, + "grad_norm": 26.88004493713379, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.38864219188690186, + "logits/rejected": -0.38103652000427246, + "logps/chosen": -161.72061157226562, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -207.19436645507812, + "loss": 1.2795, + "margin_dpo/margin_mean": 37.79635238647461, + "margin_dpo/margin_std": 112.05845642089844, + "step": 156 + }, + { + "KL/chosen_KL_mean": -82.7113265991211, + "KL/mean": -125.64529418945312, + "KL/rejected_KL_mean": -168.57925415039062, + "KL/std": 93.22598266601562, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.006050461903214455, + "fcm_dpo/delta": -0.12728933990001678, + "fcm_dpo/margin": 85.86795043945312, + "fcm_dpo/q_t": 0.38134443759918213, + "grad_norm": 22.28666877746582, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.3800089955329895, + "logits/rejected": -0.38667839765548706, + "logps/chosen": -144.07977294921875, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -276.22564697265625, + "loss": 1.0333, + "margin_dpo/margin_mean": 85.8679428100586, + "margin_dpo/margin_std": 106.39141082763672, + "step": 157 + }, + { + "KL/chosen_KL_mean": -83.55116271972656, + "KL/mean": -128.82968139648438, + "KL/rejected_KL_mean": -174.10821533203125, + "KL/std": 115.12596130371094, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.00591618474572897, + "fcm_dpo/delta": -0.14340424537658691, + "fcm_dpo/margin": 90.55704498291016, + "fcm_dpo/q_t": 0.3894592523574829, + "grad_norm": 18.08319854736328, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.38433006405830383, + "logits/rejected": -0.4023793339729309, + "logps/chosen": -141.16409301757812, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -287.8028869628906, + "loss": 1.0619, + "margin_dpo/margin_mean": 90.55704498291016, + "margin_dpo/margin_std": 139.39837646484375, + "step": 158 + }, + { + "KL/chosen_KL_mean": -94.66435241699219, + "KL/mean": -125.72183227539062, + "KL/rejected_KL_mean": -156.77931213378906, + "KL/std": 101.91374969482422, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.00585651770234108, + "fcm_dpo/delta": -0.025087930262088776, + "fcm_dpo/margin": 62.11495590209961, + "fcm_dpo/q_t": 0.41839802265167236, + "grad_norm": 20.296737670898438, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.3827086091041565, + "logits/rejected": -0.3514580726623535, + "logps/chosen": -176.22470092773438, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -245.67800903320312, + "loss": 1.1554, + "margin_dpo/margin_mean": 62.114952087402344, + "margin_dpo/margin_std": 111.131103515625, + "step": 159 + }, + { + "KL/chosen_KL_mean": -96.57306671142578, + "KL/mean": -138.43612670898438, + "KL/rejected_KL_mean": -180.2991943359375, + "KL/std": 108.08401489257812, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.005739990621805191, + "fcm_dpo/delta": -0.08555498719215393, + "fcm_dpo/margin": 83.72612762451172, + "fcm_dpo/q_t": 0.39850178360939026, + "grad_norm": 22.248849868774414, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.33730852603912354, + "logits/rejected": -0.3274146020412445, + "logps/chosen": -162.303955078125, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -277.51702880859375, + "loss": 1.0949, + "margin_dpo/margin_mean": 83.72611999511719, + "margin_dpo/margin_std": 135.13731384277344, + "step": 160 + }, + { + "KL/chosen_KL_mean": -81.85186004638672, + "KL/mean": -118.49143981933594, + "KL/rejected_KL_mean": -155.1310272216797, + "KL/std": 85.41311645507812, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.005724776070564985, + "fcm_dpo/delta": -0.020363179966807365, + "fcm_dpo/margin": 73.2791748046875, + "fcm_dpo/q_t": 0.406619131565094, + "grad_norm": 20.848581314086914, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.3932560682296753, + "logits/rejected": -0.39453092217445374, + "logps/chosen": -134.288330078125, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -238.56198120117188, + "loss": 1.0969, + "margin_dpo/margin_mean": 73.2791748046875, + "margin_dpo/margin_std": 107.4331283569336, + "step": 161 + }, + { + "KL/chosen_KL_mean": -79.50802612304688, + "KL/mean": -115.2374038696289, + "KL/rejected_KL_mean": -150.966796875, + "KL/std": 94.02556610107422, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.005622171796858311, + "fcm_dpo/delta": -0.08570829033851624, + "fcm_dpo/margin": 71.4587631225586, + "fcm_dpo/q_t": 0.4103834927082062, + "grad_norm": 20.780170440673828, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.39732295274734497, + "logits/rejected": -0.380290687084198, + "logps/chosen": -142.1186065673828, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -240.35736083984375, + "loss": 1.1175, + "margin_dpo/margin_mean": 71.4587631225586, + "margin_dpo/margin_std": 111.37112426757812, + "step": 162 + }, + { + "KL/chosen_KL_mean": -89.01995849609375, + "KL/mean": -124.2052230834961, + "KL/rejected_KL_mean": -159.3905029296875, + "KL/std": 94.75924682617188, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.00562655832618475, + "fcm_dpo/delta": 0.004222155548632145, + "fcm_dpo/margin": 70.37054443359375, + "fcm_dpo/q_t": 0.412727952003479, + "grad_norm": 19.10055923461914, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.3235231637954712, + "logits/rejected": -0.30375292897224426, + "logps/chosen": -144.04159545898438, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -234.8087158203125, + "loss": 1.1219, + "margin_dpo/margin_mean": 70.37054443359375, + "margin_dpo/margin_std": 113.79344177246094, + "step": 163 + }, + { + "KL/chosen_KL_mean": -80.56571197509766, + "KL/mean": -123.41600799560547, + "KL/rejected_KL_mean": -166.26629638671875, + "KL/std": 91.62100219726562, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.005524798296391964, + "fcm_dpo/delta": -0.07855250686407089, + "fcm_dpo/margin": 85.70059204101562, + "fcm_dpo/q_t": 0.392604798078537, + "grad_norm": 20.88271141052246, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.34821730852127075, + "logits/rejected": -0.3345373868942261, + "logps/chosen": -136.20639038085938, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -245.93093872070312, + "loss": 1.0468, + "margin_dpo/margin_mean": 85.7005844116211, + "margin_dpo/margin_std": 105.46710968017578, + "step": 164 + }, + { + "KL/chosen_KL_mean": -87.90548706054688, + "KL/mean": -115.72706604003906, + "KL/rejected_KL_mean": -143.54864501953125, + "KL/std": 78.20730590820312, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.005559276789426804, + "fcm_dpo/delta": 0.01808040775358677, + "fcm_dpo/margin": 55.64316177368164, + "fcm_dpo/q_t": 0.4298439621925354, + "grad_norm": 21.801048278808594, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.374223530292511, + "logits/rejected": -0.3485214114189148, + "logps/chosen": -149.2161865234375, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -217.21925354003906, + "loss": 1.1802, + "margin_dpo/margin_mean": 55.643165588378906, + "margin_dpo/margin_std": 104.16834259033203, + "step": 165 + }, + { + "KL/chosen_KL_mean": -77.63679504394531, + "KL/mean": -129.28445434570312, + "KL/rejected_KL_mean": -180.93209838867188, + "KL/std": 113.89814758300781, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.005399251356720924, + "fcm_dpo/delta": -0.16811595857143402, + "fcm_dpo/margin": 103.29530334472656, + "fcm_dpo/q_t": 0.38437381386756897, + "grad_norm": 16.782482147216797, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.30714818835258484, + "logits/rejected": -0.31717637181282043, + "logps/chosen": -128.62039184570312, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -279.0272216796875, + "loss": 1.0276, + "margin_dpo/margin_mean": 103.29530334472656, + "margin_dpo/margin_std": 143.48902893066406, + "step": 166 + }, + { + "KL/chosen_KL_mean": -78.55130004882812, + "KL/mean": -132.80894470214844, + "KL/rejected_KL_mean": -187.06658935546875, + "KL/std": 103.52567291259766, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.005233749747276306, + "fcm_dpo/delta": -0.1781093180179596, + "fcm_dpo/margin": 108.51528930664062, + "fcm_dpo/q_t": 0.3725237250328064, + "grad_norm": 19.92547035217285, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.29969602823257446, + "logits/rejected": -0.3019357919692993, + "logps/chosen": -128.9753875732422, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -283.0970153808594, + "loss": 0.9849, + "margin_dpo/margin_mean": 108.51528930664062, + "margin_dpo/margin_std": 119.91732025146484, + "step": 167 + }, + { + "KL/chosen_KL_mean": -85.6337890625, + "KL/mean": -125.45841979980469, + "KL/rejected_KL_mean": -165.28305053710938, + "KL/std": 97.41622924804688, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.005167778581380844, + "fcm_dpo/delta": -0.012210663408041, + "fcm_dpo/margin": 79.64924621582031, + "fcm_dpo/q_t": 0.4072931706905365, + "grad_norm": 18.608476638793945, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.33297234773635864, + "logits/rejected": -0.32413923740386963, + "logps/chosen": -135.09661865234375, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -240.5915985107422, + "loss": 1.0886, + "margin_dpo/margin_mean": 79.64924621582031, + "margin_dpo/margin_std": 109.08097839355469, + "step": 168 + }, + { + "KL/chosen_KL_mean": -88.36959075927734, + "KL/mean": -123.29864501953125, + "KL/rejected_KL_mean": -158.2277069091797, + "KL/std": 96.24781799316406, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.005108817480504513, + "fcm_dpo/delta": -0.06406478583812714, + "fcm_dpo/margin": 69.85810089111328, + "fcm_dpo/q_t": 0.4224638044834137, + "grad_norm": 20.606666564941406, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.3534125089645386, + "logits/rejected": -0.33774369955062866, + "logps/chosen": -148.17303466796875, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -241.57345581054688, + "loss": 1.1671, + "margin_dpo/margin_mean": 69.85810089111328, + "margin_dpo/margin_std": 132.85946655273438, + "step": 169 + }, + { + "KL/chosen_KL_mean": -78.21940612792969, + "KL/mean": -118.97434997558594, + "KL/rejected_KL_mean": -159.7292938232422, + "KL/std": 91.10197448730469, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.005068926140666008, + "fcm_dpo/delta": -0.014479998499155045, + "fcm_dpo/margin": 81.50988006591797, + "fcm_dpo/q_t": 0.40496397018432617, + "grad_norm": 17.198488235473633, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.3058898448944092, + "logits/rejected": -0.2954203486442566, + "logps/chosen": -127.69117736816406, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -235.64663696289062, + "loss": 1.0891, + "margin_dpo/margin_mean": 81.50987243652344, + "margin_dpo/margin_std": 109.02605438232422, + "step": 170 + }, + { + "KL/chosen_KL_mean": -113.6211929321289, + "KL/mean": -146.87875366210938, + "KL/rejected_KL_mean": -180.1363067626953, + "KL/std": 106.15757751464844, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.005119826178997755, + "fcm_dpo/delta": 0.025242332369089127, + "fcm_dpo/margin": 66.5151138305664, + "fcm_dpo/q_t": 0.42743390798568726, + "grad_norm": 38.00779724121094, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.3356257677078247, + "logits/rejected": -0.31839603185653687, + "logps/chosen": -198.12051391601562, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -289.5184020996094, + "loss": 1.1811, + "margin_dpo/margin_mean": 66.5151138305664, + "margin_dpo/margin_std": 136.447998046875, + "step": 171 + }, + { + "KL/chosen_KL_mean": -99.87997436523438, + "KL/mean": -134.4633026123047, + "KL/rejected_KL_mean": -169.046630859375, + "KL/std": 102.62731170654297, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.0051492988131940365, + "fcm_dpo/delta": 0.04548470675945282, + "fcm_dpo/margin": 69.16663360595703, + "fcm_dpo/q_t": 0.4197036027908325, + "grad_norm": 22.4017333984375, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.3853462338447571, + "logits/rejected": -0.3706569969654083, + "logps/chosen": -168.53390502929688, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -254.48329162597656, + "loss": 1.1428, + "margin_dpo/margin_mean": 69.16664123535156, + "margin_dpo/margin_std": 115.39752197265625, + "step": 172 + }, + { + "KL/chosen_KL_mean": -91.83029174804688, + "KL/mean": -126.89036560058594, + "KL/rejected_KL_mean": -161.950439453125, + "KL/std": 97.12596130371094, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.005159324966371059, + "fcm_dpo/delta": -0.014127220027148724, + "fcm_dpo/margin": 70.12013244628906, + "fcm_dpo/q_t": 0.4173567295074463, + "grad_norm": 22.549245834350586, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.40207815170288086, + "logits/rejected": -0.37598925828933716, + "logps/chosen": -154.88116455078125, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -240.63436889648438, + "loss": 1.1204, + "margin_dpo/margin_mean": 70.1201400756836, + "margin_dpo/margin_std": 101.39967346191406, + "step": 173 + }, + { + "KL/chosen_KL_mean": -87.19264221191406, + "KL/mean": -129.03155517578125, + "KL/rejected_KL_mean": -170.87046813964844, + "KL/std": 101.7507553100586, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.00515699852257967, + "fcm_dpo/delta": -0.03346613794565201, + "fcm_dpo/margin": 83.67782592773438, + "fcm_dpo/q_t": 0.4046477675437927, + "grad_norm": 20.429983139038086, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.32683736085891724, + "logits/rejected": -0.34450677037239075, + "logps/chosen": -140.55560302734375, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -272.78167724609375, + "loss": 1.0933, + "margin_dpo/margin_mean": 83.67782592773438, + "margin_dpo/margin_std": 121.98257446289062, + "step": 174 + }, + { + "KL/chosen_KL_mean": -80.11123657226562, + "KL/mean": -137.241455078125, + "KL/rejected_KL_mean": -194.37167358398438, + "KL/std": 108.67372131347656, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.00497650308534503, + "fcm_dpo/delta": -0.17907479405403137, + "fcm_dpo/margin": 114.26045227050781, + "fcm_dpo/q_t": 0.3701305389404297, + "grad_norm": 26.741565704345703, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.31531715393066406, + "logits/rejected": -0.3027985394001007, + "logps/chosen": -125.52899932861328, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -283.87750244140625, + "loss": 0.9685, + "margin_dpo/margin_mean": 114.26045227050781, + "margin_dpo/margin_std": 114.41642761230469, + "step": 175 + }, + { + "KL/chosen_KL_mean": -87.97651672363281, + "KL/mean": -136.24281311035156, + "KL/rejected_KL_mean": -184.50909423828125, + "KL/std": 110.33782958984375, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.0048902323469519615, + "fcm_dpo/delta": -0.07556910812854767, + "fcm_dpo/margin": 96.5325927734375, + "fcm_dpo/q_t": 0.39659416675567627, + "grad_norm": 19.591796875, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.28238004446029663, + "logits/rejected": -0.2996824383735657, + "logps/chosen": -138.42935180664062, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -280.06805419921875, + "loss": 1.0562, + "margin_dpo/margin_mean": 96.5325927734375, + "margin_dpo/margin_std": 128.50119018554688, + "step": 176 + }, + { + "KL/chosen_KL_mean": -100.68170166015625, + "KL/mean": -148.4861297607422, + "KL/rejected_KL_mean": -196.29055786132812, + "KL/std": 117.41094970703125, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.004829715471714735, + "fcm_dpo/delta": -0.06472522020339966, + "fcm_dpo/margin": 95.6088638305664, + "fcm_dpo/q_t": 0.39729976654052734, + "grad_norm": 24.398744583129883, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.33544689416885376, + "logits/rejected": -0.3281004726886749, + "logps/chosen": -161.89816284179688, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -292.184326171875, + "loss": 1.0578, + "margin_dpo/margin_mean": 95.6088638305664, + "margin_dpo/margin_std": 125.45742797851562, + "step": 177 + }, + { + "KL/chosen_KL_mean": -110.39865112304688, + "KL/mean": -170.1486358642578, + "KL/rejected_KL_mean": -229.89862060546875, + "KL/std": 136.63662719726562, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.004665023647248745, + "fcm_dpo/delta": -0.16771014034748077, + "fcm_dpo/margin": 119.49998474121094, + "fcm_dpo/q_t": 0.378621906042099, + "grad_norm": 27.116313934326172, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.22689053416252136, + "logits/rejected": -0.21835270524024963, + "logps/chosen": -168.66343688964844, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -335.26397705078125, + "loss": 1.0089, + "margin_dpo/margin_mean": 119.49998474121094, + "margin_dpo/margin_std": 149.8529815673828, + "step": 178 + }, + { + "KL/chosen_KL_mean": -114.35818481445312, + "KL/mean": -156.54830932617188, + "KL/rejected_KL_mean": -198.7384033203125, + "KL/std": 118.86895751953125, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.0046033794060349464, + "fcm_dpo/delta": -0.042461097240448, + "fcm_dpo/margin": 84.38021850585938, + "fcm_dpo/q_t": 0.41394948959350586, + "grad_norm": 27.796348571777344, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.24278169870376587, + "logits/rejected": -0.25100645422935486, + "logps/chosen": -175.41650390625, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -289.2662353515625, + "loss": 1.1443, + "margin_dpo/margin_mean": 84.38021087646484, + "margin_dpo/margin_std": 148.56832885742188, + "step": 179 + }, + { + "KL/chosen_KL_mean": -98.7266845703125, + "KL/mean": -152.715576171875, + "KL/rejected_KL_mean": -206.70445251464844, + "KL/std": 105.4239273071289, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.004534706007689238, + "fcm_dpo/delta": -0.09437774121761322, + "fcm_dpo/margin": 107.97775268554688, + "fcm_dpo/q_t": 0.38854917883872986, + "grad_norm": 20.505590438842773, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.2942737638950348, + "logits/rejected": -0.29070383310317993, + "logps/chosen": -153.06939697265625, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -304.9162902832031, + "loss": 1.0374, + "margin_dpo/margin_mean": 107.97775268554688, + "margin_dpo/margin_std": 132.716064453125, + "step": 180 + }, + { + "KL/chosen_KL_mean": -84.4170913696289, + "KL/mean": -117.78306579589844, + "KL/rejected_KL_mean": -151.1490478515625, + "KL/std": 96.01103210449219, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.004534834064543247, + "fcm_dpo/delta": 0.03865887597203255, + "fcm_dpo/margin": 66.7319564819336, + "fcm_dpo/q_t": 0.4305010437965393, + "grad_norm": 18.200300216674805, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.2900279462337494, + "logits/rejected": -0.26291823387145996, + "logps/chosen": -139.41754150390625, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -212.80520629882812, + "loss": 1.1788, + "margin_dpo/margin_mean": 66.7319564819336, + "margin_dpo/margin_std": 122.24834442138672, + "step": 181 + }, + { + "KL/chosen_KL_mean": -80.32139587402344, + "KL/mean": -139.32687377929688, + "KL/rejected_KL_mean": -198.33236694335938, + "KL/std": 116.76557159423828, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.004447543527930975, + "fcm_dpo/delta": -0.1325186789035797, + "fcm_dpo/margin": 118.0109634399414, + "fcm_dpo/q_t": 0.38049906492233276, + "grad_norm": 17.205862045288086, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.2654987573623657, + "logits/rejected": -0.27521929144859314, + "logps/chosen": -121.42924499511719, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -287.85394287109375, + "loss": 1.0195, + "margin_dpo/margin_mean": 118.0109634399414, + "margin_dpo/margin_std": 140.99806213378906, + "step": 182 + }, + { + "KL/chosen_KL_mean": -119.13544464111328, + "KL/mean": -150.522705078125, + "KL/rejected_KL_mean": -181.90997314453125, + "KL/std": 96.85575103759766, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.004403113387525082, + "fcm_dpo/delta": -0.03415660560131073, + "fcm_dpo/margin": 62.77452850341797, + "fcm_dpo/q_t": 0.43539807200431824, + "grad_norm": 19.68757438659668, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.2988319993019104, + "logits/rejected": -0.2892191410064697, + "logps/chosen": -176.66000366210938, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -257.88568115234375, + "loss": 1.1893, + "margin_dpo/margin_mean": 62.7745246887207, + "margin_dpo/margin_std": 106.36762237548828, + "step": 183 + }, + { + "KL/chosen_KL_mean": -99.12240600585938, + "KL/mean": -132.2347412109375, + "KL/rejected_KL_mean": -165.3470458984375, + "KL/std": 91.38697052001953, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.00440495740622282, + "fcm_dpo/delta": 0.0041860174387693405, + "fcm_dpo/margin": 66.22463989257812, + "fcm_dpo/q_t": 0.43339359760284424, + "grad_norm": 17.60099983215332, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.3370361328125, + "logits/rejected": -0.32724446058273315, + "logps/chosen": -157.6673583984375, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -241.98110961914062, + "loss": 1.1785, + "margin_dpo/margin_mean": 66.22463989257812, + "margin_dpo/margin_std": 115.2151870727539, + "step": 184 + }, + { + "KL/chosen_KL_mean": -106.90892791748047, + "KL/mean": -133.06826782226562, + "KL/rejected_KL_mean": -159.2276153564453, + "KL/std": 104.17123413085938, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.004406801424920559, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 52.31868362426758, + "fcm_dpo/q_t": 0.4490601420402527, + "grad_norm": 17.461320877075195, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.30607613921165466, + "logits/rejected": -0.28324705362319946, + "logps/chosen": -168.93478393554688, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -232.99020385742188, + "loss": 1.2391, + "margin_dpo/margin_mean": 52.31868362426758, + "margin_dpo/margin_std": 125.27377319335938, + "step": 185 + }, + { + "KL/chosen_KL_mean": -98.97467041015625, + "KL/mean": -148.86280822753906, + "KL/rejected_KL_mean": -198.75091552734375, + "KL/std": 106.32249450683594, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.004376481287181377, + "fcm_dpo/delta": -0.03841752931475639, + "fcm_dpo/margin": 99.7762680053711, + "fcm_dpo/q_t": 0.40035003423690796, + "grad_norm": 24.914920806884766, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.36513596773147583, + "logits/rejected": -0.3440634608268738, + "logps/chosen": -168.32814025878906, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -286.8233642578125, + "loss": 1.0628, + "margin_dpo/margin_mean": 99.7762680053711, + "margin_dpo/margin_std": 122.89549255371094, + "step": 186 + }, + { + "KL/chosen_KL_mean": -92.30848693847656, + "KL/mean": -134.05665588378906, + "KL/rejected_KL_mean": -175.8048095703125, + "KL/std": 101.3410415649414, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.004364049062132835, + "fcm_dpo/delta": -0.02067478746175766, + "fcm_dpo/margin": 83.49632263183594, + "fcm_dpo/q_t": 0.4153994023799896, + "grad_norm": 20.103660583496094, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.32625746726989746, + "logits/rejected": -0.32030683755874634, + "logps/chosen": -145.06494140625, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -257.77392578125, + "loss": 1.1099, + "margin_dpo/margin_mean": 83.49632263183594, + "margin_dpo/margin_std": 109.70829772949219, + "step": 187 + }, + { + "KL/chosen_KL_mean": -87.06753540039062, + "KL/mean": -136.92221069335938, + "KL/rejected_KL_mean": -186.77688598632812, + "KL/std": 112.0629653930664, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.004336144309490919, + "fcm_dpo/delta": -0.033802639693021774, + "fcm_dpo/margin": 99.7093505859375, + "fcm_dpo/q_t": 0.4015769958496094, + "grad_norm": 23.71269989013672, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.2950665056705475, + "logits/rejected": -0.29936039447784424, + "logps/chosen": -136.4830322265625, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -276.31732177734375, + "loss": 1.0653, + "margin_dpo/margin_mean": 99.7093505859375, + "margin_dpo/margin_std": 123.11599731445312, + "step": 188 + }, + { + "KL/chosen_KL_mean": -101.37373352050781, + "KL/mean": -143.13787841796875, + "KL/rejected_KL_mean": -184.90200805664062, + "KL/std": 113.1136474609375, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.0043535660952329636, + "fcm_dpo/delta": 0.03768228366971016, + "fcm_dpo/margin": 83.52828979492188, + "fcm_dpo/q_t": 0.4195026755332947, + "grad_norm": 23.715402603149414, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.31556832790374756, + "logits/rejected": -0.2963595986366272, + "logps/chosen": -153.772705078125, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -257.0693664550781, + "loss": 1.1365, + "margin_dpo/margin_mean": 83.52828979492188, + "margin_dpo/margin_std": 139.05917358398438, + "step": 189 + }, + { + "KL/chosen_KL_mean": -109.61285400390625, + "KL/mean": -158.9443359375, + "KL/rejected_KL_mean": -208.27578735351562, + "KL/std": 121.74542236328125, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.004317188635468483, + "fcm_dpo/delta": -0.09547873586416245, + "fcm_dpo/margin": 98.66294860839844, + "fcm_dpo/q_t": 0.40431755781173706, + "grad_norm": 17.334806442260742, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.3407072424888611, + "logits/rejected": -0.3281744122505188, + "logps/chosen": -174.29592895507812, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -310.8263244628906, + "loss": 1.0965, + "margin_dpo/margin_mean": 98.66294860839844, + "margin_dpo/margin_std": 141.34219360351562, + "step": 190 + }, + { + "KL/chosen_KL_mean": -102.64785766601562, + "KL/mean": -174.55918884277344, + "KL/rejected_KL_mean": -246.47052001953125, + "KL/std": 142.98800659179688, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.004130992107093334, + "fcm_dpo/delta": -0.20744846761226654, + "fcm_dpo/margin": 143.82266235351562, + "fcm_dpo/q_t": 0.36818498373031616, + "grad_norm": 17.989526748657227, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.32365429401397705, + "logits/rejected": -0.3008269667625427, + "logps/chosen": -171.30673217773438, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -356.6101989746094, + "loss": 0.9687, + "margin_dpo/margin_mean": 143.82266235351562, + "margin_dpo/margin_std": 157.698974609375, + "step": 191 + }, + { + "KL/chosen_KL_mean": -133.00082397460938, + "KL/mean": -180.19398498535156, + "KL/rejected_KL_mean": -227.38717651367188, + "KL/std": 124.70111846923828, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.004118871875107288, + "fcm_dpo/delta": 0.011550931259989738, + "fcm_dpo/margin": 94.38633728027344, + "fcm_dpo/q_t": 0.4139176309108734, + "grad_norm": 23.6864013671875, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.2887781262397766, + "logits/rejected": -0.28792911767959595, + "logps/chosen": -202.72772216796875, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -330.70849609375, + "loss": 1.1258, + "margin_dpo/margin_mean": 94.38633728027344, + "margin_dpo/margin_std": 153.73789978027344, + "step": 192 + }, + { + "KL/chosen_KL_mean": -134.04782104492188, + "KL/mean": -161.41824340820312, + "KL/rejected_KL_mean": -188.78866577148438, + "KL/std": 115.35488891601562, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.00413061399012804, + "fcm_dpo/delta": 0.04584140703082085, + "fcm_dpo/margin": 54.740840911865234, + "fcm_dpo/q_t": 0.44635239243507385, + "grad_norm": 25.42896270751953, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.3118895888328552, + "logits/rejected": -0.3038848042488098, + "logps/chosen": -194.23831176757812, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -265.19622802734375, + "loss": 1.2611, + "margin_dpo/margin_mean": 54.740840911865234, + "margin_dpo/margin_std": 146.91921997070312, + "step": 193 + }, + { + "KL/chosen_KL_mean": -84.5526351928711, + "KL/mean": -129.1312255859375, + "KL/rejected_KL_mean": -173.70980834960938, + "KL/std": 95.83538818359375, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.004156617913395166, + "fcm_dpo/delta": 0.030460838228464127, + "fcm_dpo/margin": 89.15718078613281, + "fcm_dpo/q_t": 0.41309183835983276, + "grad_norm": 20.639873504638672, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.22054284811019897, + "logits/rejected": -0.20486515760421753, + "logps/chosen": -122.39300537109375, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -234.39459228515625, + "loss": 1.0959, + "margin_dpo/margin_mean": 89.15718078613281, + "margin_dpo/margin_std": 105.14234924316406, + "step": 194 + }, + { + "KL/chosen_KL_mean": -132.57052612304688, + "KL/mean": -180.29159545898438, + "KL/rejected_KL_mean": -228.01266479492188, + "KL/std": 119.44116973876953, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.004175318870693445, + "fcm_dpo/delta": 0.0015607781242579222, + "fcm_dpo/margin": 95.4421157836914, + "fcm_dpo/q_t": 0.40840816497802734, + "grad_norm": 23.395151138305664, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.2661496102809906, + "logits/rejected": -0.28478342294692993, + "logps/chosen": -187.46209716796875, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -324.78363037109375, + "loss": 1.0887, + "margin_dpo/margin_mean": 95.4421157836914, + "margin_dpo/margin_std": 123.58929443359375, + "step": 195 + }, + { + "KL/chosen_KL_mean": -103.19564819335938, + "KL/mean": -157.65101623535156, + "KL/rejected_KL_mean": -212.10638427734375, + "KL/std": 119.3145523071289, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.00412184651941061, + "fcm_dpo/delta": -0.052077874541282654, + "fcm_dpo/margin": 108.91070556640625, + "fcm_dpo/q_t": 0.39910075068473816, + "grad_norm": 17.112943649291992, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.2943457067012787, + "logits/rejected": -0.27971380949020386, + "logps/chosen": -156.44088745117188, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -288.1593322753906, + "loss": 1.0719, + "margin_dpo/margin_mean": 108.91070556640625, + "margin_dpo/margin_std": 143.52066040039062, + "step": 196 + }, + { + "KL/chosen_KL_mean": -104.69943237304688, + "KL/mean": -146.47988891601562, + "KL/rejected_KL_mean": -188.2603759765625, + "KL/std": 104.27262878417969, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.004131827969104052, + "fcm_dpo/delta": -0.0026854458265006542, + "fcm_dpo/margin": 83.56092834472656, + "fcm_dpo/q_t": 0.42058151960372925, + "grad_norm": 20.54574966430664, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.2778225243091583, + "logits/rejected": -0.2714378833770752, + "logps/chosen": -165.11976623535156, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -265.4692687988281, + "loss": 1.1277, + "margin_dpo/margin_mean": 83.56092834472656, + "margin_dpo/margin_std": 118.95997619628906, + "step": 197 + }, + { + "KL/chosen_KL_mean": -116.94888305664062, + "KL/mean": -171.30592346191406, + "KL/rejected_KL_mean": -225.6629638671875, + "KL/std": 129.6251678466797, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.004103054292500019, + "fcm_dpo/delta": -0.048192791640758514, + "fcm_dpo/margin": 108.71409606933594, + "fcm_dpo/q_t": 0.4009076654911041, + "grad_norm": 20.30980110168457, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.3229246735572815, + "logits/rejected": -0.32260096073150635, + "logps/chosen": -171.98507690429688, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -322.9062194824219, + "loss": 1.07, + "margin_dpo/margin_mean": 108.71409606933594, + "margin_dpo/margin_std": 147.5482635498047, + "step": 198 + }, + { + "KL/chosen_KL_mean": -111.7033462524414, + "KL/mean": -165.23153686523438, + "KL/rejected_KL_mean": -218.7597198486328, + "KL/std": 117.17296600341797, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.0040468741208314896, + "fcm_dpo/delta": -0.03558361902832985, + "fcm_dpo/margin": 107.05636596679688, + "fcm_dpo/q_t": 0.4017881453037262, + "grad_norm": 20.354114532470703, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.32916444540023804, + "logits/rejected": -0.3120754361152649, + "logps/chosen": -168.53216552734375, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -303.4079284667969, + "loss": 1.0799, + "margin_dpo/margin_mean": 107.05636596679688, + "margin_dpo/margin_std": 143.97113037109375, + "step": 199 + }, + { + "KL/chosen_KL_mean": -109.22659301757812, + "KL/mean": -162.36294555664062, + "KL/rejected_KL_mean": -215.49929809570312, + "KL/std": 125.6125259399414, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.004030941985547543, + "fcm_dpo/delta": -0.029960284009575844, + "fcm_dpo/margin": 106.272705078125, + "fcm_dpo/q_t": 0.40496158599853516, + "grad_norm": 20.5546875, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.2713373303413391, + "logits/rejected": -0.2532021701335907, + "logps/chosen": -162.29364013671875, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -296.10772705078125, + "loss": 1.1004, + "margin_dpo/margin_mean": 106.272705078125, + "margin_dpo/margin_std": 162.5040740966797, + "step": 200 + }, + { + "KL/chosen_KL_mean": -115.20796203613281, + "KL/mean": -168.23330688476562, + "KL/rejected_KL_mean": -221.25863647460938, + "KL/std": 133.3345947265625, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.0040308814495801926, + "fcm_dpo/delta": -0.028764937072992325, + "fcm_dpo/margin": 106.05069732666016, + "fcm_dpo/q_t": 0.40355366468429565, + "grad_norm": 19.861337661743164, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.3617175817489624, + "logits/rejected": -0.3581845164299011, + "logps/chosen": -190.6101837158203, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -336.0668640136719, + "loss": 1.0884, + "margin_dpo/margin_mean": 106.05069732666016, + "margin_dpo/margin_std": 150.36595153808594, + "step": 201 + }, + { + "KL/chosen_KL_mean": -117.26287078857422, + "KL/mean": -154.33970642089844, + "KL/rejected_KL_mean": -191.4165496826172, + "KL/std": 113.34939575195312, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.004033088684082031, + "fcm_dpo/delta": 0.0457109659910202, + "fcm_dpo/margin": 74.15367126464844, + "fcm_dpo/q_t": 0.43285977840423584, + "grad_norm": 19.15420150756836, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.28320997953414917, + "logits/rejected": -0.2973800003528595, + "logps/chosen": -167.36419677734375, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -278.4015808105469, + "loss": 1.1911, + "margin_dpo/margin_mean": 74.15367889404297, + "margin_dpo/margin_std": 146.57894897460938, + "step": 202 + }, + { + "KL/chosen_KL_mean": -117.84786987304688, + "KL/mean": -164.06817626953125, + "KL/rejected_KL_mean": -210.28848266601562, + "KL/std": 117.28266906738281, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.004062448628246784, + "fcm_dpo/delta": 0.0254225991666317, + "fcm_dpo/margin": 92.44062042236328, + "fcm_dpo/q_t": 0.4140324890613556, + "grad_norm": 20.89508819580078, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.31334608793258667, + "logits/rejected": -0.30196499824523926, + "logps/chosen": -178.4575653076172, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -296.1844482421875, + "loss": 1.1102, + "margin_dpo/margin_mean": 92.44061279296875, + "margin_dpo/margin_std": 129.81788635253906, + "step": 203 + }, + { + "KL/chosen_KL_mean": -130.2074432373047, + "KL/mean": -172.8050537109375, + "KL/rejected_KL_mean": -215.40267944335938, + "KL/std": 128.60855102539062, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.004102942533791065, + "fcm_dpo/delta": 0.05228898674249649, + "fcm_dpo/margin": 85.19523620605469, + "fcm_dpo/q_t": 0.42274272441864014, + "grad_norm": 23.4987850189209, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.42270368337631226, + "logits/rejected": -0.38482266664505005, + "logps/chosen": -210.37240600585938, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -303.09857177734375, + "loss": 1.1561, + "margin_dpo/margin_mean": 85.19524383544922, + "margin_dpo/margin_std": 150.52664184570312, + "step": 204 + }, + { + "KL/chosen_KL_mean": -125.3318099975586, + "KL/mean": -181.85325622558594, + "KL/rejected_KL_mean": -238.37469482421875, + "KL/std": 129.90475463867188, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.004025366622954607, + "fcm_dpo/delta": -0.11009746789932251, + "fcm_dpo/margin": 113.04287719726562, + "fcm_dpo/q_t": 0.3948434889316559, + "grad_norm": 19.743913650512695, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.3243464231491089, + "logits/rejected": -0.29430297017097473, + "logps/chosen": -184.716552734375, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -323.499755859375, + "loss": 1.0638, + "margin_dpo/margin_mean": 113.04287719726562, + "margin_dpo/margin_std": 142.10345458984375, + "step": 205 + }, + { + "KL/chosen_KL_mean": -114.53091430664062, + "KL/mean": -175.703125, + "KL/rejected_KL_mean": -236.8753204345703, + "KL/std": 121.82723236083984, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.003958613146096468, + "fcm_dpo/delta": -0.089159294962883, + "fcm_dpo/margin": 122.34440612792969, + "fcm_dpo/q_t": 0.38700979948043823, + "grad_norm": 30.70645523071289, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.23936225473880768, + "logits/rejected": -0.2412106692790985, + "logps/chosen": -161.49542236328125, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -335.82879638671875, + "loss": 1.0249, + "margin_dpo/margin_mean": 122.34440612792969, + "margin_dpo/margin_std": 133.05758666992188, + "step": 206 + }, + { + "KL/chosen_KL_mean": -105.94319152832031, + "KL/mean": -176.26629638671875, + "KL/rejected_KL_mean": -246.58941650390625, + "KL/std": 140.27398681640625, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.0038703742902725935, + "fcm_dpo/delta": -0.15251129865646362, + "fcm_dpo/margin": 140.64620971679688, + "fcm_dpo/q_t": 0.3791235089302063, + "grad_norm": 22.952560424804688, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.36327940225601196, + "logits/rejected": -0.3385031223297119, + "logps/chosen": -161.99945068359375, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -331.0372314453125, + "loss": 1.0008, + "margin_dpo/margin_mean": 140.64620971679688, + "margin_dpo/margin_std": 163.15628051757812, + "step": 207 + }, + { + "KL/chosen_KL_mean": -164.36953735351562, + "KL/mean": -219.13931274414062, + "KL/rejected_KL_mean": -273.90911865234375, + "KL/std": 135.67556762695312, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.0038177226670086384, + "fcm_dpo/delta": -0.01908494532108307, + "fcm_dpo/margin": 109.53958129882812, + "fcm_dpo/q_t": 0.40520548820495605, + "grad_norm": 25.327423095703125, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.34415963292121887, + "logits/rejected": -0.32587701082229614, + "logps/chosen": -231.4371337890625, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -368.19598388671875, + "loss": 1.1045, + "margin_dpo/margin_mean": 109.5395736694336, + "margin_dpo/margin_std": 166.34190368652344, + "step": 208 + }, + { + "KL/chosen_KL_mean": -139.50582885742188, + "KL/mean": -187.45782470703125, + "KL/rejected_KL_mean": -235.40982055664062, + "KL/std": 121.63459777832031, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.003841415047645569, + "fcm_dpo/delta": 0.03278043121099472, + "fcm_dpo/margin": 95.90400695800781, + "fcm_dpo/q_t": 0.41615039110183716, + "grad_norm": 26.806434631347656, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.23338031768798828, + "logits/rejected": -0.22082915902137756, + "logps/chosen": -195.68751525878906, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -316.3513488769531, + "loss": 1.1346, + "margin_dpo/margin_mean": 95.90400695800781, + "margin_dpo/margin_std": 152.61289978027344, + "step": 209 + }, + { + "KL/chosen_KL_mean": -126.15152740478516, + "KL/mean": -182.57559204101562, + "KL/rejected_KL_mean": -238.99969482421875, + "KL/std": 126.32806396484375, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.0038408541586250067, + "fcm_dpo/delta": -0.03512062132358551, + "fcm_dpo/margin": 112.84817504882812, + "fcm_dpo/q_t": 0.40204623341560364, + "grad_norm": 24.104612350463867, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.2401820421218872, + "logits/rejected": -0.23000288009643555, + "logps/chosen": -172.52334594726562, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -315.6813049316406, + "loss": 1.0781, + "margin_dpo/margin_mean": 112.84817504882812, + "margin_dpo/margin_std": 151.28884887695312, + "step": 210 + }, + { + "KL/chosen_KL_mean": -181.4200439453125, + "KL/mean": -229.34503173828125, + "KL/rejected_KL_mean": -277.27001953125, + "KL/std": 144.36581420898438, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.003842352656647563, + "fcm_dpo/delta": 0.0328550860285759, + "fcm_dpo/margin": 95.84998321533203, + "fcm_dpo/q_t": 0.4207329750061035, + "grad_norm": 35.524505615234375, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.3115510940551758, + "logits/rejected": -0.2750454545021057, + "logps/chosen": -260.3523864746094, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -364.09100341796875, + "loss": 1.1518, + "margin_dpo/margin_mean": 95.8499755859375, + "margin_dpo/margin_std": 170.39910888671875, + "step": 211 + }, + { + "KL/chosen_KL_mean": -148.82797241210938, + "KL/mean": -218.71502685546875, + "KL/rejected_KL_mean": -288.6020812988281, + "KL/std": 157.48464965820312, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.003741275053471327, + "fcm_dpo/delta": -0.13187746703624725, + "fcm_dpo/margin": 139.7740936279297, + "fcm_dpo/q_t": 0.3831748962402344, + "grad_norm": 24.796751022338867, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.2529842257499695, + "logits/rejected": -0.24166236817836761, + "logps/chosen": -207.02499389648438, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -391.659912109375, + "loss": 1.0357, + "margin_dpo/margin_mean": 139.7740936279297, + "margin_dpo/margin_std": 174.7883758544922, + "step": 212 + }, + { + "KL/chosen_KL_mean": -139.20504760742188, + "KL/mean": -205.1407470703125, + "KL/rejected_KL_mean": -271.0764465332031, + "KL/std": 135.7279815673828, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.003681202419102192, + "fcm_dpo/delta": -0.09037460386753082, + "fcm_dpo/margin": 131.87139892578125, + "fcm_dpo/q_t": 0.38955849409103394, + "grad_norm": 25.563806533813477, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.2899351119995117, + "logits/rejected": -0.26357612013816833, + "logps/chosen": -206.71774291992188, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -364.99114990234375, + "loss": 1.0417, + "margin_dpo/margin_mean": 131.87139892578125, + "margin_dpo/margin_std": 162.48764038085938, + "step": 213 + }, + { + "KL/chosen_KL_mean": -122.16487884521484, + "KL/mean": -181.35964965820312, + "KL/rejected_KL_mean": -240.554443359375, + "KL/std": 135.08131408691406, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.003657899796962738, + "fcm_dpo/delta": -0.0345565602183342, + "fcm_dpo/margin": 118.38955688476562, + "fcm_dpo/q_t": 0.40235117077827454, + "grad_norm": 23.318286895751953, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.2528512477874756, + "logits/rejected": -0.25968700647354126, + "logps/chosen": -163.769775390625, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -318.07183837890625, + "loss": 1.076, + "margin_dpo/margin_mean": 118.38955688476562, + "margin_dpo/margin_std": 157.97145080566406, + "step": 214 + }, + { + "KL/chosen_KL_mean": -141.74395751953125, + "KL/mean": -205.91880798339844, + "KL/rejected_KL_mean": -270.0936279296875, + "KL/std": 142.14804077148438, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.003605864243581891, + "fcm_dpo/delta": -0.06620416045188904, + "fcm_dpo/margin": 128.34967041015625, + "fcm_dpo/q_t": 0.39388930797576904, + "grad_norm": 22.542652130126953, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.27835413813591003, + "logits/rejected": -0.2704225182533264, + "logps/chosen": -195.02322387695312, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -360.05828857421875, + "loss": 1.042, + "margin_dpo/margin_mean": 128.34967041015625, + "margin_dpo/margin_std": 147.6060791015625, + "step": 215 + }, + { + "KL/chosen_KL_mean": -142.92184448242188, + "KL/mean": -205.51254272460938, + "KL/rejected_KL_mean": -268.103271484375, + "KL/std": 144.8072509765625, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.0035881579387933016, + "fcm_dpo/delta": -0.05153839662671089, + "fcm_dpo/margin": 125.18144226074219, + "fcm_dpo/q_t": 0.3995622396469116, + "grad_norm": 24.884077072143555, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.25770103931427, + "logits/rejected": -0.24090343713760376, + "logps/chosen": -191.80963134765625, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -345.30218505859375, + "loss": 1.0816, + "margin_dpo/margin_mean": 125.18144226074219, + "margin_dpo/margin_std": 178.65567016601562, + "step": 216 + }, + { + "KL/chosen_KL_mean": -149.1703338623047, + "KL/mean": -222.16607666015625, + "KL/rejected_KL_mean": -295.1618347167969, + "KL/std": 151.5289306640625, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.0034975060261785984, + "fcm_dpo/delta": -0.11692032217979431, + "fcm_dpo/margin": 145.99148559570312, + "fcm_dpo/q_t": 0.38548392057418823, + "grad_norm": 21.3267765045166, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.24890464544296265, + "logits/rejected": -0.24278412759304047, + "logps/chosen": -199.01564025878906, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -395.24017333984375, + "loss": 1.0206, + "margin_dpo/margin_mean": 145.9915008544922, + "margin_dpo/margin_std": 172.6138458251953, + "step": 217 + }, + { + "KL/chosen_KL_mean": -155.8583984375, + "KL/mean": -212.3345947265625, + "KL/rejected_KL_mean": -268.810791015625, + "KL/std": 147.9781494140625, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.003482023486867547, + "fcm_dpo/delta": 0.006924496963620186, + "fcm_dpo/margin": 112.95238494873047, + "fcm_dpo/q_t": 0.41226643323898315, + "grad_norm": 21.803455352783203, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.2691187858581543, + "logits/rejected": -0.26593974232673645, + "logps/chosen": -214.43508911132812, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -356.65716552734375, + "loss": 1.118, + "margin_dpo/margin_mean": 112.95237731933594, + "margin_dpo/margin_std": 175.06222534179688, + "step": 218 + }, + { + "KL/chosen_KL_mean": -165.19070434570312, + "KL/mean": -214.21315002441406, + "KL/rejected_KL_mean": -263.235595703125, + "KL/std": 151.88267517089844, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.0034828686621040106, + "fcm_dpo/delta": -0.018037430942058563, + "fcm_dpo/margin": 98.04489135742188, + "fcm_dpo/q_t": 0.4232712686061859, + "grad_norm": 27.709285736083984, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.2986528277397156, + "logits/rejected": -0.29288095235824585, + "logps/chosen": -226.27456665039062, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -349.0660400390625, + "loss": 1.172, + "margin_dpo/margin_mean": 98.04489135742188, + "margin_dpo/margin_std": 185.69854736328125, + "step": 219 + }, + { + "KL/chosen_KL_mean": -186.55886840820312, + "KL/mean": -226.88809204101562, + "KL/rejected_KL_mean": -267.21728515625, + "KL/std": 130.2073974609375, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.0035033172462135553, + "fcm_dpo/delta": 0.03829696774482727, + "fcm_dpo/margin": 80.6584243774414, + "fcm_dpo/q_t": 0.4332135319709778, + "grad_norm": 25.699840545654297, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.2736346125602722, + "logits/rejected": -0.24952954053878784, + "logps/chosen": -256.59014892578125, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -354.9028015136719, + "loss": 1.1748, + "margin_dpo/margin_mean": 80.6584243774414, + "margin_dpo/margin_std": 131.5479736328125, + "step": 220 + }, + { + "KL/chosen_KL_mean": -158.74429321289062, + "KL/mean": -247.2379150390625, + "KL/rejected_KL_mean": -335.7315673828125, + "KL/std": 162.83950805664062, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.0034025218337774277, + "fcm_dpo/delta": -0.21474148333072662, + "fcm_dpo/margin": 176.98724365234375, + "fcm_dpo/q_t": 0.3616037666797638, + "grad_norm": 24.72466278076172, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.2757573127746582, + "logits/rejected": -0.2824522852897644, + "logps/chosen": -210.89901733398438, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -442.19921875, + "loss": 0.9467, + "margin_dpo/margin_mean": 176.98724365234375, + "margin_dpo/margin_std": 165.9235076904297, + "step": 221 + }, + { + "KL/chosen_KL_mean": -162.6530303955078, + "KL/mean": -229.3107452392578, + "KL/rejected_KL_mean": -295.96844482421875, + "KL/std": 149.646484375, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.0033454755321145058, + "fcm_dpo/delta": -0.0482868030667305, + "fcm_dpo/margin": 133.31539916992188, + "fcm_dpo/q_t": 0.39826542139053345, + "grad_norm": 19.118408203125, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.3239976763725281, + "logits/rejected": -0.3265075087547302, + "logps/chosen": -223.62413024902344, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -395.9696044921875, + "loss": 1.0628, + "margin_dpo/margin_mean": 133.31539916992188, + "margin_dpo/margin_std": 167.6547393798828, + "step": 222 + }, + { + "KL/chosen_KL_mean": -172.12124633789062, + "KL/mean": -228.33541870117188, + "KL/rejected_KL_mean": -284.54962158203125, + "KL/std": 145.0126953125, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.0033332279417663813, + "fcm_dpo/delta": 0.02623838558793068, + "fcm_dpo/margin": 112.42835998535156, + "fcm_dpo/q_t": 0.415128618478775, + "grad_norm": 22.70659637451172, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.22877205908298492, + "logits/rejected": -0.2223329246044159, + "logps/chosen": -224.76181030273438, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -367.3746337890625, + "loss": 1.1327, + "margin_dpo/margin_mean": 112.42835998535156, + "margin_dpo/margin_std": 183.301025390625, + "step": 223 + }, + { + "KL/chosen_KL_mean": -146.13272094726562, + "KL/mean": -210.43267822265625, + "KL/rejected_KL_mean": -274.73260498046875, + "KL/std": 158.8013153076172, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.0032867477275431156, + "fcm_dpo/delta": -0.0822690799832344, + "fcm_dpo/margin": 128.5998992919922, + "fcm_dpo/q_t": 0.40540385246276855, + "grad_norm": 25.003032684326172, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.2951328754425049, + "logits/rejected": -0.27578508853912354, + "logps/chosen": -194.72813415527344, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -351.84912109375, + "loss": 1.0916, + "margin_dpo/margin_mean": 128.5998992919922, + "margin_dpo/margin_std": 182.40701293945312, + "step": 224 + }, + { + "KL/chosen_KL_mean": -164.68719482421875, + "KL/mean": -240.1545867919922, + "KL/rejected_KL_mean": -315.6219482421875, + "KL/std": 152.24942016601562, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.0032383957877755165, + "fcm_dpo/delta": -0.09345349669456482, + "fcm_dpo/margin": 150.93475341796875, + "fcm_dpo/q_t": 0.3889888525009155, + "grad_norm": 23.60213851928711, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.3080984950065613, + "logits/rejected": -0.2962522804737091, + "logps/chosen": -222.6876678466797, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -415.5248718261719, + "loss": 1.0359, + "margin_dpo/margin_mean": 150.9347686767578, + "margin_dpo/margin_std": 183.36630249023438, + "step": 225 + }, + { + "KL/chosen_KL_mean": -138.68136596679688, + "KL/mean": -197.8099365234375, + "KL/rejected_KL_mean": -256.93853759765625, + "KL/std": 146.85330200195312, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.003195089753717184, + "fcm_dpo/delta": -0.0479632206261158, + "fcm_dpo/margin": 118.25714111328125, + "fcm_dpo/q_t": 0.4144435524940491, + "grad_norm": 28.424585342407227, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.3738971948623657, + "logits/rejected": -0.3347446918487549, + "logps/chosen": -197.58016967773438, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -335.62628173828125, + "loss": 1.1141, + "margin_dpo/margin_mean": 118.25714111328125, + "margin_dpo/margin_std": 170.03778076171875, + "step": 226 + }, + { + "KL/chosen_KL_mean": -165.69725036621094, + "KL/mean": -239.54896545410156, + "KL/rejected_KL_mean": -313.4006652832031, + "KL/std": 168.34571838378906, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.003151601180434227, + "fcm_dpo/delta": -0.06917452812194824, + "fcm_dpo/margin": 147.70343017578125, + "fcm_dpo/q_t": 0.3932172954082489, + "grad_norm": 23.378986358642578, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.37774959206581116, + "logits/rejected": -0.3662099242210388, + "logps/chosen": -224.76943969726562, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -412.81304931640625, + "loss": 1.0437, + "margin_dpo/margin_mean": 147.70343017578125, + "margin_dpo/margin_std": 175.08055114746094, + "step": 227 + }, + { + "KL/chosen_KL_mean": -172.15579223632812, + "KL/mean": -225.4064178466797, + "KL/rejected_KL_mean": -278.65704345703125, + "KL/std": 140.27630615234375, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.0031857825815677643, + "fcm_dpo/delta": 0.06269420683383942, + "fcm_dpo/margin": 106.50125885009766, + "fcm_dpo/q_t": 0.4218415915966034, + "grad_norm": 22.29404640197754, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.3632548451423645, + "logits/rejected": -0.34667009115219116, + "logps/chosen": -238.04708862304688, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -369.705810546875, + "loss": 1.1458, + "margin_dpo/margin_mean": 106.50126647949219, + "margin_dpo/margin_std": 171.11866760253906, + "step": 228 + }, + { + "KL/chosen_KL_mean": -177.75022888183594, + "KL/mean": -235.13406372070312, + "KL/rejected_KL_mean": -292.5179138183594, + "KL/std": 161.8447265625, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.003208862617611885, + "fcm_dpo/delta": 0.03290688246488571, + "fcm_dpo/margin": 114.76766204833984, + "fcm_dpo/q_t": 0.4150307774543762, + "grad_norm": 27.36089324951172, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.39479750394821167, + "logits/rejected": -0.3608490228652954, + "logps/chosen": -248.45660400390625, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -377.0453186035156, + "loss": 1.12, + "margin_dpo/margin_mean": 114.76766967773438, + "margin_dpo/margin_std": 168.88572692871094, + "step": 229 + }, + { + "KL/chosen_KL_mean": -138.62490844726562, + "KL/mean": -225.5928955078125, + "KL/rejected_KL_mean": -312.56085205078125, + "KL/std": 155.33326721191406, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.0031352953519672155, + "fcm_dpo/delta": -0.15396267175674438, + "fcm_dpo/margin": 173.93594360351562, + "fcm_dpo/q_t": 0.37328383326530457, + "grad_norm": 31.360973358154297, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.33708709478378296, + "logits/rejected": -0.3414233326911926, + "logps/chosen": -177.90692138671875, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -398.1827697753906, + "loss": 0.9793, + "margin_dpo/margin_mean": 173.93594360351562, + "margin_dpo/margin_std": 170.8446807861328, + "step": 230 + }, + { + "KL/chosen_KL_mean": -159.44564819335938, + "KL/mean": -219.74270629882812, + "KL/rejected_KL_mean": -280.0397644042969, + "KL/std": 139.70201110839844, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.0031266703736037016, + "fcm_dpo/delta": 0.023831568658351898, + "fcm_dpo/margin": 120.59412384033203, + "fcm_dpo/q_t": 0.41457289457321167, + "grad_norm": 23.459474563598633, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.35875609517097473, + "logits/rejected": -0.3278740644454956, + "logps/chosen": -222.72207641601562, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -354.16363525390625, + "loss": 1.1048, + "margin_dpo/margin_mean": 120.59412384033203, + "margin_dpo/margin_std": 163.7926788330078, + "step": 231 + }, + { + "KL/chosen_KL_mean": -196.00889587402344, + "KL/mean": -243.2371826171875, + "KL/rejected_KL_mean": -290.4654541015625, + "KL/std": 160.13824462890625, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.0031496628653258085, + "fcm_dpo/delta": 0.0321061946451664, + "fcm_dpo/margin": 94.45652770996094, + "fcm_dpo/q_t": 0.43301987648010254, + "grad_norm": 23.38385772705078, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.3741375505924225, + "logits/rejected": -0.35106799006462097, + "logps/chosen": -266.7576599121094, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -374.4425048828125, + "loss": 1.1733, + "margin_dpo/margin_mean": 94.45652770996094, + "margin_dpo/margin_std": 164.90455627441406, + "step": 232 + }, + { + "KL/chosen_KL_mean": -178.70001220703125, + "KL/mean": -255.507080078125, + "KL/rejected_KL_mean": -332.31414794921875, + "KL/std": 172.16485595703125, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.0031155627220869064, + "fcm_dpo/delta": -0.08247057348489761, + "fcm_dpo/margin": 153.6141357421875, + "fcm_dpo/q_t": 0.39588814973831177, + "grad_norm": 29.239364624023438, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.3583585023880005, + "logits/rejected": -0.36010071635246277, + "logps/chosen": -233.58294677734375, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -439.7942199707031, + "loss": 1.074, + "margin_dpo/margin_mean": 153.6141357421875, + "margin_dpo/margin_std": 224.79373168945312, + "step": 233 + }, + { + "KL/chosen_KL_mean": -166.96612548828125, + "KL/mean": -246.79653930664062, + "KL/rejected_KL_mean": -326.626953125, + "KL/std": 153.10523986816406, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.0030329639557749033, + "fcm_dpo/delta": -0.09048783034086227, + "fcm_dpo/margin": 159.66079711914062, + "fcm_dpo/q_t": 0.38900816440582275, + "grad_norm": 22.883272171020508, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.3432962894439697, + "logits/rejected": -0.37032586336135864, + "logps/chosen": -211.06065368652344, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -426.6335754394531, + "loss": 1.027, + "margin_dpo/margin_mean": 159.66079711914062, + "margin_dpo/margin_std": 172.949462890625, + "step": 234 + }, + { + "KL/chosen_KL_mean": -199.75088500976562, + "KL/mean": -252.1366424560547, + "KL/rejected_KL_mean": -304.52239990234375, + "KL/std": 146.66983032226562, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0030462555587291718, + "fcm_dpo/delta": 0.012087216600775719, + "fcm_dpo/margin": 104.77151489257812, + "fcm_dpo/q_t": 0.4263225793838501, + "grad_norm": 25.01669692993164, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.4110462963581085, + "logits/rejected": -0.3919840157032013, + "logps/chosen": -261.9888000488281, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -394.9174499511719, + "loss": 1.1627, + "margin_dpo/margin_mean": 104.7715072631836, + "margin_dpo/margin_std": 177.99478149414062, + "step": 235 + }, + { + "KL/chosen_KL_mean": -147.17303466796875, + "KL/mean": -230.70997619628906, + "KL/rejected_KL_mean": -314.2469177246094, + "KL/std": 156.2943115234375, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.003003060584887862, + "fcm_dpo/delta": -0.10706593096256256, + "fcm_dpo/margin": 167.07388305664062, + "fcm_dpo/q_t": 0.38220128417015076, + "grad_norm": 37.00520324707031, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.3595033288002014, + "logits/rejected": -0.3598778247833252, + "logps/chosen": -196.51438903808594, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -417.758544921875, + "loss": 0.9941, + "margin_dpo/margin_mean": 167.0738983154297, + "margin_dpo/margin_std": 150.903076171875, + "step": 236 + }, + { + "KL/chosen_KL_mean": -206.67808532714844, + "KL/mean": -268.8200988769531, + "KL/rejected_KL_mean": -330.9621276855469, + "KL/std": 161.38294982910156, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.0030013294890522957, + "fcm_dpo/delta": 0.02796686813235283, + "fcm_dpo/margin": 124.28401947021484, + "fcm_dpo/q_t": 0.4151972234249115, + "grad_norm": 30.164600372314453, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.3569568991661072, + "logits/rejected": -0.3496701419353485, + "logps/chosen": -260.84619140625, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -425.74249267578125, + "loss": 1.1212, + "margin_dpo/margin_mean": 124.28402709960938, + "margin_dpo/margin_std": 188.1182403564453, + "step": 237 + }, + { + "KL/chosen_KL_mean": -190.0740203857422, + "KL/mean": -252.8286590576172, + "KL/rejected_KL_mean": -315.58331298828125, + "KL/std": 162.52825927734375, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0029827167745679617, + "fcm_dpo/delta": -0.032333556562662125, + "fcm_dpo/margin": 125.50928497314453, + "fcm_dpo/q_t": 0.41632723808288574, + "grad_norm": 25.658233642578125, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.353759229183197, + "logits/rejected": -0.3687341809272766, + "logps/chosen": -244.04714965820312, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -405.00128173828125, + "loss": 1.1158, + "margin_dpo/margin_mean": 125.50928497314453, + "margin_dpo/margin_std": 183.60763549804688, + "step": 238 + }, + { + "KL/chosen_KL_mean": -202.21400451660156, + "KL/mean": -273.4901123046875, + "KL/rejected_KL_mean": -344.7661437988281, + "KL/std": 152.6262969970703, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.00295428279787302, + "fcm_dpo/delta": -0.02298806607723236, + "fcm_dpo/margin": 142.5521697998047, + "fcm_dpo/q_t": 0.4021487236022949, + "grad_norm": 26.8980770111084, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.37527695298194885, + "logits/rejected": -0.36356276273727417, + "logps/chosen": -260.31182861328125, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -438.3591003417969, + "loss": 1.073, + "margin_dpo/margin_mean": 142.55215454101562, + "margin_dpo/margin_std": 171.53750610351562, + "step": 239 + }, + { + "KL/chosen_KL_mean": -213.47067260742188, + "KL/mean": -273.86285400390625, + "KL/rejected_KL_mean": -334.2550048828125, + "KL/std": 169.06903076171875, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.0029616348911076784, + "fcm_dpo/delta": -0.012477724812924862, + "fcm_dpo/margin": 120.78433227539062, + "fcm_dpo/q_t": 0.418390154838562, + "grad_norm": 35.30418014526367, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.3764647841453552, + "logits/rejected": -0.35425490140914917, + "logps/chosen": -274.08514404296875, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -408.37353515625, + "loss": 1.1455, + "margin_dpo/margin_mean": 120.78433227539062, + "margin_dpo/margin_std": 201.88650512695312, + "step": 240 + }, + { + "KL/chosen_KL_mean": -177.76251220703125, + "KL/mean": -270.9705810546875, + "KL/rejected_KL_mean": -364.17864990234375, + "KL/std": 181.308837890625, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0029037785716354847, + "fcm_dpo/delta": -0.14915932714939117, + "fcm_dpo/margin": 186.41616821289062, + "fcm_dpo/q_t": 0.3788558542728424, + "grad_norm": 24.147903442382812, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.3980168104171753, + "logits/rejected": -0.37038347125053406, + "logps/chosen": -243.85357666015625, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -452.23956298828125, + "loss": 1.0009, + "margin_dpo/margin_mean": 186.41616821289062, + "margin_dpo/margin_std": 207.73675537109375, + "step": 241 + }, + { + "KL/chosen_KL_mean": -199.2550048828125, + "KL/mean": -266.1572265625, + "KL/rejected_KL_mean": -333.0594482421875, + "KL/std": 153.730224609375, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.002892129821702838, + "fcm_dpo/delta": 0.013144008815288544, + "fcm_dpo/margin": 133.804443359375, + "fcm_dpo/q_t": 0.4118873178958893, + "grad_norm": 32.67736053466797, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.3379971981048584, + "logits/rejected": -0.31009137630462646, + "logps/chosen": -267.11895751953125, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -416.4197998046875, + "loss": 1.0996, + "margin_dpo/margin_mean": 133.804443359375, + "margin_dpo/margin_std": 179.3169403076172, + "step": 242 + }, + { + "KL/chosen_KL_mean": -197.12493896484375, + "KL/mean": -276.5483703613281, + "KL/rejected_KL_mean": -355.9717712402344, + "KL/std": 169.24517822265625, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.0028545591048896313, + "fcm_dpo/delta": -0.056080713868141174, + "fcm_dpo/margin": 158.84686279296875, + "fcm_dpo/q_t": 0.3958790898323059, + "grad_norm": 23.821611404418945, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.38002443313598633, + "logits/rejected": -0.34791332483291626, + "logps/chosen": -260.20916748046875, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -432.3074035644531, + "loss": 1.0494, + "margin_dpo/margin_mean": 158.8468475341797, + "margin_dpo/margin_std": 188.71011352539062, + "step": 243 + }, + { + "KL/chosen_KL_mean": -171.1998291015625, + "KL/mean": -252.1486358642578, + "KL/rejected_KL_mean": -333.0974426269531, + "KL/std": 160.80581665039062, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.0028192391619086266, + "fcm_dpo/delta": -0.0593310110270977, + "fcm_dpo/margin": 161.8976287841797, + "fcm_dpo/q_t": 0.3936727046966553, + "grad_norm": 41.310176849365234, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.45028799772262573, + "logits/rejected": -0.4422299861907959, + "logps/chosen": -232.34051513671875, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -427.9893798828125, + "loss": 1.0318, + "margin_dpo/margin_mean": 161.89764404296875, + "margin_dpo/margin_std": 163.90341186523438, + "step": 244 + }, + { + "KL/chosen_KL_mean": -195.34979248046875, + "KL/mean": -260.214599609375, + "KL/rejected_KL_mean": -325.07940673828125, + "KL/std": 156.43153381347656, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.0027832172345370054, + "fcm_dpo/delta": -0.0592423714697361, + "fcm_dpo/margin": 129.7296142578125, + "fcm_dpo/q_t": 0.41568508744239807, + "grad_norm": 24.933977127075195, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.4311348497867584, + "logits/rejected": -0.40182024240493774, + "logps/chosen": -262.6120910644531, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -412.719482421875, + "loss": 1.1202, + "margin_dpo/margin_mean": 129.7296142578125, + "margin_dpo/margin_std": 177.87432861328125, + "step": 245 + }, + { + "KL/chosen_KL_mean": -189.2398681640625, + "KL/mean": -255.52532958984375, + "KL/rejected_KL_mean": -321.810791015625, + "KL/std": 175.95999145507812, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.0027891253121197224, + "fcm_dpo/delta": 0.03135085850954056, + "fcm_dpo/margin": 132.5709228515625, + "fcm_dpo/q_t": 0.41607168316841125, + "grad_norm": 25.692262649536133, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.4559187889099121, + "logits/rejected": -0.4175548553466797, + "logps/chosen": -255.9368438720703, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -406.1571044921875, + "loss": 1.1143, + "margin_dpo/margin_mean": 132.5709228515625, + "margin_dpo/margin_std": 189.25132751464844, + "step": 246 + }, + { + "KL/chosen_KL_mean": -225.79959106445312, + "KL/mean": -317.54595947265625, + "KL/rejected_KL_mean": -409.2923583984375, + "KL/std": 183.09689331054688, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.0027523355092853308, + "fcm_dpo/delta": -0.11074697971343994, + "fcm_dpo/margin": 183.49273681640625, + "fcm_dpo/q_t": 0.3836245536804199, + "grad_norm": 29.499662399291992, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.3989511728286743, + "logits/rejected": -0.39488452672958374, + "logps/chosen": -282.4049377441406, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -515.5855712890625, + "loss": 1.0132, + "margin_dpo/margin_mean": 183.4927215576172, + "margin_dpo/margin_std": 198.01431274414062, + "step": 247 + }, + { + "KL/chosen_KL_mean": -196.19500732421875, + "KL/mean": -277.23980712890625, + "KL/rejected_KL_mean": -358.2845458984375, + "KL/std": 151.759033203125, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.0027115982957184315, + "fcm_dpo/delta": -0.04182609170675278, + "fcm_dpo/margin": 162.08953857421875, + "fcm_dpo/q_t": 0.3955652117729187, + "grad_norm": 23.417442321777344, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.38095736503601074, + "logits/rejected": -0.39841145277023315, + "logps/chosen": -240.2382354736328, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -450.14141845703125, + "loss": 1.0341, + "margin_dpo/margin_mean": 162.08953857421875, + "margin_dpo/margin_std": 148.43768310546875, + "step": 248 + }, + { + "KL/chosen_KL_mean": -257.7716369628906, + "KL/mean": -302.58721923828125, + "KL/rejected_KL_mean": -347.4028015136719, + "KL/std": 170.12164306640625, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0027165513020008802, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 89.63117218017578, + "fcm_dpo/q_t": 0.44409990310668945, + "grad_norm": 37.337493896484375, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.44070059061050415, + "logits/rejected": -0.4142151176929474, + "logps/chosen": -320.2139892578125, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -427.870849609375, + "loss": 1.2304, + "margin_dpo/margin_mean": 89.63116455078125, + "margin_dpo/margin_std": 203.44784545898438, + "step": 249 + }, + { + "KL/chosen_KL_mean": -228.001708984375, + "KL/mean": -310.7402038574219, + "KL/rejected_KL_mean": -393.47869873046875, + "KL/std": 170.05862426757812, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.002707940526306629, + "fcm_dpo/delta": -0.0505223274230957, + "fcm_dpo/margin": 165.47698974609375, + "fcm_dpo/q_t": 0.3952521085739136, + "grad_norm": 38.479984283447266, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.41971248388290405, + "logits/rejected": -0.36163565516471863, + "logps/chosen": -293.638427734375, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -467.3505554199219, + "loss": 1.0428, + "margin_dpo/margin_mean": 165.47698974609375, + "margin_dpo/margin_std": 176.21603393554688, + "step": 250 + }, + { + "KL/chosen_KL_mean": -238.53073120117188, + "KL/mean": -293.2124328613281, + "KL/rejected_KL_mean": -347.8941650390625, + "KL/std": 176.7196502685547, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.0027050368953496218, + "fcm_dpo/delta": 0.029284248128533363, + "fcm_dpo/margin": 109.36344909667969, + "fcm_dpo/q_t": 0.4319899082183838, + "grad_norm": 31.399150848388672, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.41108593344688416, + "logits/rejected": -0.3802783787250519, + "logps/chosen": -295.71343994140625, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -425.5576171875, + "loss": 1.1815, + "margin_dpo/margin_mean": 109.36344909667969, + "margin_dpo/margin_std": 193.80047607421875, + "step": 251 + }, + { + "KL/chosen_KL_mean": -231.04844665527344, + "KL/mean": -306.7342834472656, + "KL/rejected_KL_mean": -382.4200744628906, + "KL/std": 153.58529663085938, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.002697393763810396, + "fcm_dpo/delta": -0.008723736740648746, + "fcm_dpo/margin": 151.37161254882812, + "fcm_dpo/q_t": 0.402085542678833, + "grad_norm": 22.914670944213867, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.4410606026649475, + "logits/rejected": -0.3991231322288513, + "logps/chosen": -302.7341003417969, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -467.1780700683594, + "loss": 1.062, + "margin_dpo/margin_mean": 151.37161254882812, + "margin_dpo/margin_std": 155.73104858398438, + "step": 252 + }, + { + "KL/chosen_KL_mean": -195.75592041015625, + "KL/mean": -271.3820495605469, + "KL/rejected_KL_mean": -347.0081481933594, + "KL/std": 165.71835327148438, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0027113882824778557, + "fcm_dpo/delta": -0.011142361909151077, + "fcm_dpo/margin": 151.25222778320312, + "fcm_dpo/q_t": 0.40395694971084595, + "grad_norm": 22.0101318359375, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.43955349922180176, + "logits/rejected": -0.4316529929637909, + "logps/chosen": -264.8898620605469, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -445.710693359375, + "loss": 1.0797, + "margin_dpo/margin_mean": 151.25222778320312, + "margin_dpo/margin_std": 183.66358947753906, + "step": 253 + }, + { + "KL/chosen_KL_mean": -194.18955993652344, + "KL/mean": -260.004150390625, + "KL/rejected_KL_mean": -325.8187561035156, + "KL/std": 175.55361938476562, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.00270917359739542, + "fcm_dpo/delta": 0.04502296447753906, + "fcm_dpo/margin": 131.6291961669922, + "fcm_dpo/q_t": 0.42085546255111694, + "grad_norm": 25.79990005493164, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.38691574335098267, + "logits/rejected": -0.368805468082428, + "logps/chosen": -248.3445587158203, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -416.12640380859375, + "loss": 1.1425, + "margin_dpo/margin_mean": 131.6291961669922, + "margin_dpo/margin_std": 221.32101440429688, + "step": 254 + }, + { + "KL/chosen_KL_mean": -192.90646362304688, + "KL/mean": -261.3782653808594, + "KL/rejected_KL_mean": -329.85009765625, + "KL/std": 151.24462890625, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.002720620483160019, + "fcm_dpo/delta": 0.028336577117443085, + "fcm_dpo/margin": 136.94363403320312, + "fcm_dpo/q_t": 0.4135192036628723, + "grad_norm": 22.284894943237305, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.36700326204299927, + "logits/rejected": -0.3556111454963684, + "logps/chosen": -250.04812622070312, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -420.0586853027344, + "loss": 1.1144, + "margin_dpo/margin_mean": 136.94363403320312, + "margin_dpo/margin_std": 192.08795166015625, + "step": 255 + }, + { + "KL/chosen_KL_mean": -170.9013214111328, + "KL/mean": -244.374755859375, + "KL/rejected_KL_mean": -317.84820556640625, + "KL/std": 163.259765625, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.0027340268716216087, + "fcm_dpo/delta": -0.001830903347581625, + "fcm_dpo/margin": 146.94686889648438, + "fcm_dpo/q_t": 0.40705880522727966, + "grad_norm": 21.34122085571289, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.3065924644470215, + "logits/rejected": -0.31275248527526855, + "logps/chosen": -226.0648193359375, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -410.4111022949219, + "loss": 1.0866, + "margin_dpo/margin_mean": 146.94686889648438, + "margin_dpo/margin_std": 188.91909790039062, + "step": 256 + }, + { + "KL/chosen_KL_mean": -167.6283721923828, + "KL/mean": -244.84832763671875, + "KL/rejected_KL_mean": -322.0682678222656, + "KL/std": 170.49884033203125, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.0026939453091472387, + "fcm_dpo/delta": -0.07220666855573654, + "fcm_dpo/margin": 154.43991088867188, + "fcm_dpo/q_t": 0.4040963053703308, + "grad_norm": 21.323671340942383, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.3410765528678894, + "logits/rejected": -0.32743754982948303, + "logps/chosen": -217.05206298828125, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -401.606201171875, + "loss": 1.0807, + "margin_dpo/margin_mean": 154.4398956298828, + "margin_dpo/margin_std": 192.8297119140625, + "step": 257 + }, + { + "KL/chosen_KL_mean": -220.85940551757812, + "KL/mean": -296.29345703125, + "KL/rejected_KL_mean": -371.7275390625, + "KL/std": 179.92108154296875, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.002696289913728833, + "fcm_dpo/delta": -0.0071342457085847855, + "fcm_dpo/margin": 150.86813354492188, + "fcm_dpo/q_t": 0.405239999294281, + "grad_norm": 22.390037536621094, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.316403865814209, + "logits/rejected": -0.31249940395355225, + "logps/chosen": -280.2435302734375, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -467.7176513671875, + "loss": 1.098, + "margin_dpo/margin_mean": 150.86813354492188, + "margin_dpo/margin_std": 213.5443115234375, + "step": 258 + }, + { + "KL/chosen_KL_mean": -205.83718872070312, + "KL/mean": -268.58819580078125, + "KL/rejected_KL_mean": -331.3392028808594, + "KL/std": 170.47787475585938, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.0026900002267211676, + "fcm_dpo/delta": -0.00019304518355056643, + "fcm_dpo/margin": 125.50199890136719, + "fcm_dpo/q_t": 0.42243772745132446, + "grad_norm": 20.560476303100586, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.4018911123275757, + "logits/rejected": -0.39738091826438904, + "logps/chosen": -258.66552734375, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -420.5308532714844, + "loss": 1.1405, + "margin_dpo/margin_mean": 125.50199890136719, + "margin_dpo/margin_std": 191.87994384765625, + "step": 259 + }, + { + "KL/chosen_KL_mean": -208.1739501953125, + "KL/mean": -294.4159851074219, + "KL/rejected_KL_mean": -380.65802001953125, + "KL/std": 178.80706787109375, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.002675027120858431, + "fcm_dpo/delta": -0.06443756073713303, + "fcm_dpo/margin": 172.48403930664062, + "fcm_dpo/q_t": 0.3951517343521118, + "grad_norm": 27.486682891845703, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.38799139857292175, + "logits/rejected": -0.40423405170440674, + "logps/chosen": -255.5916290283203, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -475.747802734375, + "loss": 1.0375, + "margin_dpo/margin_mean": 172.48403930664062, + "margin_dpo/margin_std": 194.35281372070312, + "step": 260 + }, + { + "KL/chosen_KL_mean": -217.77267456054688, + "KL/mean": -297.56304931640625, + "KL/rejected_KL_mean": -377.35345458984375, + "KL/std": 191.00912475585938, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0026403923984616995, + "fcm_dpo/delta": -0.022464631125330925, + "fcm_dpo/margin": 159.58074951171875, + "fcm_dpo/q_t": 0.40387552976608276, + "grad_norm": 20.6318359375, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.32272404432296753, + "logits/rejected": -0.3193345069885254, + "logps/chosen": -270.8040771484375, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -465.868408203125, + "loss": 1.0811, + "margin_dpo/margin_mean": 159.58074951171875, + "margin_dpo/margin_std": 211.976318359375, + "step": 261 + }, + { + "KL/chosen_KL_mean": -250.58642578125, + "KL/mean": -317.03765869140625, + "KL/rejected_KL_mean": -383.4888916015625, + "KL/std": 174.0106964111328, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.0026418212801218033, + "fcm_dpo/delta": -0.0069253211840987206, + "fcm_dpo/margin": 132.9025115966797, + "fcm_dpo/q_t": 0.4175712466239929, + "grad_norm": 24.307924270629883, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.36420363187789917, + "logits/rejected": -0.3351730406284332, + "logps/chosen": -310.20654296875, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -469.9074401855469, + "loss": 1.1143, + "margin_dpo/margin_mean": 132.9025115966797, + "margin_dpo/margin_std": 171.23260498046875, + "step": 262 + }, + { + "KL/chosen_KL_mean": -230.8678741455078, + "KL/mean": -323.4657287597656, + "KL/rejected_KL_mean": -416.06353759765625, + "KL/std": 206.47894287109375, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.0025957219768315554, + "fcm_dpo/delta": -0.08551076054573059, + "fcm_dpo/margin": 185.19569396972656, + "fcm_dpo/q_t": 0.38983917236328125, + "grad_norm": 21.63703155517578, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.3600388169288635, + "logits/rejected": -0.3340407907962799, + "logps/chosen": -290.288818359375, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -512.9207763671875, + "loss": 1.0386, + "margin_dpo/margin_mean": 185.1956787109375, + "margin_dpo/margin_std": 220.6270751953125, + "step": 263 + }, + { + "KL/chosen_KL_mean": -235.51724243164062, + "KL/mean": -316.014404296875, + "KL/rejected_KL_mean": -396.51153564453125, + "KL/std": 189.37460327148438, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.0025572467129677534, + "fcm_dpo/delta": -0.07371211796998978, + "fcm_dpo/margin": 160.99427795410156, + "fcm_dpo/q_t": 0.4057387709617615, + "grad_norm": 26.796667098999023, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.4365754723548889, + "logits/rejected": -0.42901504039764404, + "logps/chosen": -298.23931884765625, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -490.36773681640625, + "loss": 1.09, + "margin_dpo/margin_mean": 160.9942626953125, + "margin_dpo/margin_std": 215.35891723632812, + "step": 264 + }, + { + "KL/chosen_KL_mean": -261.9628601074219, + "KL/mean": -337.67828369140625, + "KL/rejected_KL_mean": -413.39373779296875, + "KL/std": 211.77447509765625, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.002561673056334257, + "fcm_dpo/delta": 0.012572679668664932, + "fcm_dpo/margin": 151.43084716796875, + "fcm_dpo/q_t": 0.41336870193481445, + "grad_norm": 26.191164016723633, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.3544113039970398, + "logits/rejected": -0.3390767574310303, + "logps/chosen": -323.934326171875, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -501.41436767578125, + "loss": 1.1309, + "margin_dpo/margin_mean": 151.43084716796875, + "margin_dpo/margin_std": 252.63531494140625, + "step": 265 + }, + { + "KL/chosen_KL_mean": -268.99359130859375, + "KL/mean": -327.73828125, + "KL/rejected_KL_mean": -386.48297119140625, + "KL/std": 175.15029907226562, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0025742575526237488, + "fcm_dpo/delta": 0.041070494800806046, + "fcm_dpo/margin": 117.48941040039062, + "fcm_dpo/q_t": 0.4305131733417511, + "grad_norm": 49.08354949951172, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.3949073553085327, + "logits/rejected": -0.3593337833881378, + "logps/chosen": -336.09326171875, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -454.4542236328125, + "loss": 1.1722, + "margin_dpo/margin_mean": 117.48941040039062, + "margin_dpo/margin_std": 204.8365936279297, + "step": 266 + }, + { + "KL/chosen_KL_mean": -233.81716918945312, + "KL/mean": -309.85455322265625, + "KL/rejected_KL_mean": -385.8919982910156, + "KL/std": 194.2411651611328, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.0025796634145081043, + "fcm_dpo/delta": 0.007831782102584839, + "fcm_dpo/margin": 152.07479858398438, + "fcm_dpo/q_t": 0.410673052072525, + "grad_norm": 35.01557922363281, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.3717952370643616, + "logits/rejected": -0.3414815366268158, + "logps/chosen": -302.7879333496094, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -476.0604248046875, + "loss": 1.1073, + "margin_dpo/margin_mean": 152.07479858398438, + "margin_dpo/margin_std": 221.85324096679688, + "step": 267 + }, + { + "KL/chosen_KL_mean": -242.22633361816406, + "KL/mean": -315.8835144042969, + "KL/rejected_KL_mean": -389.54071044921875, + "KL/std": 183.14663696289062, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0025895836297422647, + "fcm_dpo/delta": 0.019164325669407845, + "fcm_dpo/margin": 147.3143310546875, + "fcm_dpo/q_t": 0.4133493900299072, + "grad_norm": 31.963598251342773, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.38117945194244385, + "logits/rejected": -0.38725900650024414, + "logps/chosen": -298.12664794921875, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -491.1883239746094, + "loss": 1.1165, + "margin_dpo/margin_mean": 147.3143310546875, + "margin_dpo/margin_std": 222.01901245117188, + "step": 268 + }, + { + "KL/chosen_KL_mean": -251.50296020507812, + "KL/mean": -339.5428466796875, + "KL/rejected_KL_mean": -427.58270263671875, + "KL/std": 185.965087890625, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.002580434549599886, + "fcm_dpo/delta": -0.056910403072834015, + "fcm_dpo/margin": 176.07977294921875, + "fcm_dpo/q_t": 0.396186888217926, + "grad_norm": 24.962482452392578, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.42665889859199524, + "logits/rejected": -0.4133855104446411, + "logps/chosen": -321.5425109863281, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -534.93212890625, + "loss": 1.0664, + "margin_dpo/margin_mean": 176.07977294921875, + "margin_dpo/margin_std": 232.7220001220703, + "step": 269 + }, + { + "KL/chosen_KL_mean": -225.45660400390625, + "KL/mean": -290.9217834472656, + "KL/rejected_KL_mean": -356.386962890625, + "KL/std": 165.87762451171875, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.00257058534771204, + "fcm_dpo/delta": 0.004566457588225603, + "fcm_dpo/margin": 130.93038940429688, + "fcm_dpo/q_t": 0.42188751697540283, + "grad_norm": 44.11079406738281, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.41711604595184326, + "logits/rejected": -0.41175544261932373, + "logps/chosen": -294.9900817871094, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -466.31561279296875, + "loss": 1.1463, + "margin_dpo/margin_mean": 130.93038940429688, + "margin_dpo/margin_std": 208.61184692382812, + "step": 270 + }, + { + "KL/chosen_KL_mean": -214.58233642578125, + "KL/mean": -298.871337890625, + "KL/rejected_KL_mean": -383.1603088378906, + "KL/std": 173.6548309326172, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.0025521572679281235, + "fcm_dpo/delta": -0.03186243772506714, + "fcm_dpo/margin": 168.57797241210938, + "fcm_dpo/q_t": 0.40116429328918457, + "grad_norm": 26.021621704101562, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.35142117738723755, + "logits/rejected": -0.3353433609008789, + "logps/chosen": -271.346923828125, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -475.67413330078125, + "loss": 1.0622, + "margin_dpo/margin_mean": 168.57797241210938, + "margin_dpo/margin_std": 200.06488037109375, + "step": 271 + }, + { + "KL/chosen_KL_mean": -205.83883666992188, + "KL/mean": -311.71807861328125, + "KL/rejected_KL_mean": -417.5972900390625, + "KL/std": 194.44451904296875, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.0025133024901151657, + "fcm_dpo/delta": -0.13941721618175507, + "fcm_dpo/margin": 211.75845336914062, + "fcm_dpo/q_t": 0.37617918848991394, + "grad_norm": 29.311363220214844, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.3046724796295166, + "logits/rejected": -0.317031592130661, + "logps/chosen": -255.33599853515625, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -523.1400756835938, + "loss": 0.9855, + "margin_dpo/margin_mean": 211.75845336914062, + "margin_dpo/margin_std": 202.77157592773438, + "step": 272 + }, + { + "KL/chosen_KL_mean": -241.24679565429688, + "KL/mean": -343.16290283203125, + "KL/rejected_KL_mean": -445.0789794921875, + "KL/std": 198.67681884765625, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.0024405219592154026, + "fcm_dpo/delta": -0.1030091792345047, + "fcm_dpo/margin": 203.83218383789062, + "fcm_dpo/q_t": 0.3853374123573303, + "grad_norm": 25.411273956298828, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.3357715904712677, + "logits/rejected": -0.31793808937072754, + "logps/chosen": -304.22216796875, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -537.5775756835938, + "loss": 1.0201, + "margin_dpo/margin_mean": 203.83218383789062, + "margin_dpo/margin_std": 226.27426147460938, + "step": 273 + }, + { + "KL/chosen_KL_mean": -282.93408203125, + "KL/mean": -358.6909484863281, + "KL/rejected_KL_mean": -434.44781494140625, + "KL/std": 178.87353515625, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.00245290482416749, + "fcm_dpo/delta": 0.029144397005438805, + "fcm_dpo/margin": 151.5137176513672, + "fcm_dpo/q_t": 0.41448694467544556, + "grad_norm": 28.26287841796875, + "learning_rate": 3.75e-07, + "logits/chosen": -0.25929710268974304, + "logits/rejected": -0.24342584609985352, + "logps/chosen": -338.601806640625, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -511.7808837890625, + "loss": 1.1118, + "margin_dpo/margin_mean": 151.5137176513672, + "margin_dpo/margin_std": 209.66061401367188, + "step": 274 + }, + { + "KL/chosen_KL_mean": -225.5223388671875, + "KL/mean": -307.8089599609375, + "KL/rejected_KL_mean": -390.09564208984375, + "KL/std": 182.68939208984375, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.002454479457810521, + "fcm_dpo/delta": -0.004276936873793602, + "fcm_dpo/margin": 164.57330322265625, + "fcm_dpo/q_t": 0.40654799342155457, + "grad_norm": 28.954072952270508, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.29635703563690186, + "logits/rejected": -0.3032437562942505, + "logps/chosen": -274.1170349121094, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -483.39935302734375, + "loss": 1.0855, + "margin_dpo/margin_mean": 164.57330322265625, + "margin_dpo/margin_std": 209.363525390625, + "step": 275 + }, + { + "KL/chosen_KL_mean": -246.88002014160156, + "KL/mean": -327.800537109375, + "KL/rejected_KL_mean": -408.7210693359375, + "KL/std": 188.99180603027344, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.002444277983158827, + "fcm_dpo/delta": 0.0045502083376049995, + "fcm_dpo/margin": 161.84103393554688, + "fcm_dpo/q_t": 0.4099653363227844, + "grad_norm": 28.535573959350586, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.26886600255966187, + "logits/rejected": -0.24688804149627686, + "logps/chosen": -303.4574279785156, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -479.08673095703125, + "loss": 1.1022, + "margin_dpo/margin_mean": 161.84103393554688, + "margin_dpo/margin_std": 228.76651000976562, + "step": 276 + }, + { + "KL/chosen_KL_mean": -280.7821960449219, + "KL/mean": -364.3798828125, + "KL/rejected_KL_mean": -447.9775695800781, + "KL/std": 191.448974609375, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.002451348351314664, + "fcm_dpo/delta": -0.010382965207099915, + "fcm_dpo/margin": 167.19540405273438, + "fcm_dpo/q_t": 0.40581023693084717, + "grad_norm": 28.177669525146484, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.259786993265152, + "logits/rejected": -0.24115556478500366, + "logps/chosen": -337.05377197265625, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -540.8588256835938, + "loss": 1.0926, + "margin_dpo/margin_mean": 167.1953887939453, + "margin_dpo/margin_std": 228.89898681640625, + "step": 277 + }, + { + "KL/chosen_KL_mean": -245.23388671875, + "KL/mean": -346.21710205078125, + "KL/rejected_KL_mean": -447.2003173828125, + "KL/std": 202.91122436523438, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.0024054832756519318, + "fcm_dpo/delta": -0.09053061157464981, + "fcm_dpo/margin": 201.96641540527344, + "fcm_dpo/q_t": 0.3888724744319916, + "grad_norm": 22.475250244140625, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.2788286507129669, + "logits/rejected": -0.28598934412002563, + "logps/chosen": -298.17584228515625, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -538.453857421875, + "loss": 1.0274, + "margin_dpo/margin_mean": 201.96641540527344, + "margin_dpo/margin_std": 228.17782592773438, + "step": 278 + }, + { + "KL/chosen_KL_mean": -281.48052978515625, + "KL/mean": -378.7762451171875, + "KL/rejected_KL_mean": -476.07196044921875, + "KL/std": 213.51382446289062, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.0023632331285625696, + "fcm_dpo/delta": -0.06384158134460449, + "fcm_dpo/margin": 194.59141540527344, + "fcm_dpo/q_t": 0.3963664174079895, + "grad_norm": 34.28653335571289, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.19345436990261078, + "logits/rejected": -0.1934255212545395, + "logps/chosen": -330.1218566894531, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -563.9234008789062, + "loss": 1.0656, + "margin_dpo/margin_mean": 194.59141540527344, + "margin_dpo/margin_std": 258.36968994140625, + "step": 279 + }, + { + "KL/chosen_KL_mean": -277.8961181640625, + "KL/mean": -375.35321044921875, + "KL/rejected_KL_mean": -472.810302734375, + "KL/std": 194.73277282714844, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.002347635105252266, + "fcm_dpo/delta": -0.06036704033613205, + "fcm_dpo/margin": 194.91415405273438, + "fcm_dpo/q_t": 0.39296674728393555, + "grad_norm": 31.282169342041016, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.2672705352306366, + "logits/rejected": -0.2702743411064148, + "logps/chosen": -336.6932373046875, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -571.4291381835938, + "loss": 1.0382, + "margin_dpo/margin_mean": 194.91415405273438, + "margin_dpo/margin_std": 211.66049194335938, + "step": 280 + }, + { + "KL/chosen_KL_mean": -254.6139678955078, + "KL/mean": -345.259765625, + "KL/rejected_KL_mean": -435.905517578125, + "KL/std": 198.7908477783203, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0023387479595839977, + "fcm_dpo/delta": -0.025144033133983612, + "fcm_dpo/margin": 181.29153442382812, + "fcm_dpo/q_t": 0.3999946713447571, + "grad_norm": 31.389543533325195, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.28908228874206543, + "logits/rejected": -0.2651940584182739, + "logps/chosen": -310.10247802734375, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -516.7880859375, + "loss": 1.067, + "margin_dpo/margin_mean": 181.29153442382812, + "margin_dpo/margin_std": 213.66848754882812, + "step": 281 + }, + { + "KL/chosen_KL_mean": -274.75775146484375, + "KL/mean": -341.5997619628906, + "KL/rejected_KL_mean": -408.4417724609375, + "KL/std": 199.78970336914062, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.002331382595002651, + "fcm_dpo/delta": 0.008983664214611053, + "fcm_dpo/margin": 133.68397521972656, + "fcm_dpo/q_t": 0.4300641417503357, + "grad_norm": 30.923696517944336, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.3890365958213806, + "logits/rejected": -0.37042510509490967, + "logps/chosen": -347.8279113769531, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -503.792724609375, + "loss": 1.1626, + "margin_dpo/margin_mean": 133.68397521972656, + "margin_dpo/margin_std": 228.35498046875, + "step": 282 + }, + { + "KL/chosen_KL_mean": -279.5001525878906, + "KL/mean": -386.4722595214844, + "KL/rejected_KL_mean": -493.44439697265625, + "KL/std": 235.90640258789062, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.0023081288672983646, + "fcm_dpo/delta": -0.09860023856163025, + "fcm_dpo/margin": 213.94424438476562, + "fcm_dpo/q_t": 0.38501065969467163, + "grad_norm": 30.072389602661133, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.3526265025138855, + "logits/rejected": -0.3584181070327759, + "logps/chosen": -341.3985900878906, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -590.430908203125, + "loss": 1.016, + "margin_dpo/margin_mean": 213.94424438476562, + "margin_dpo/margin_std": 222.56910705566406, + "step": 283 + }, + { + "KL/chosen_KL_mean": -256.5270690917969, + "KL/mean": -354.51007080078125, + "KL/rejected_KL_mean": -452.4930725097656, + "KL/std": 213.50137329101562, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0022687562741339207, + "fcm_dpo/delta": -0.04678437113761902, + "fcm_dpo/margin": 195.9659881591797, + "fcm_dpo/q_t": 0.39754199981689453, + "grad_norm": 29.024595260620117, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.3825564980506897, + "logits/rejected": -0.36749982833862305, + "logps/chosen": -314.96258544921875, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -545.9623413085938, + "loss": 1.0487, + "margin_dpo/margin_mean": 195.96597290039062, + "margin_dpo/margin_std": 222.738037109375, + "step": 284 + }, + { + "KL/chosen_KL_mean": -287.9412841796875, + "KL/mean": -371.3969421386719, + "KL/rejected_KL_mean": -454.8525695800781, + "KL/std": 182.3175506591797, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.002258389489725232, + "fcm_dpo/delta": -0.03637245297431946, + "fcm_dpo/margin": 166.9113006591797, + "fcm_dpo/q_t": 0.4100029170513153, + "grad_norm": 25.001317977905273, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.3526034355163574, + "logits/rejected": -0.338278204202652, + "logps/chosen": -354.1734619140625, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -553.9793701171875, + "loss": 1.0968, + "margin_dpo/margin_mean": 166.9113006591797, + "margin_dpo/margin_std": 202.8867645263672, + "step": 285 + }, + { + "KL/chosen_KL_mean": -279.0155029296875, + "KL/mean": -374.29168701171875, + "KL/rejected_KL_mean": -469.56787109375, + "KL/std": 214.52780151367188, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.002243536990135908, + "fcm_dpo/delta": -0.02873760648071766, + "fcm_dpo/margin": 190.55233764648438, + "fcm_dpo/q_t": 0.4008243978023529, + "grad_norm": 35.4481201171875, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.364835649728775, + "logits/rejected": -0.3494155704975128, + "logps/chosen": -351.9665222167969, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -558.1563110351562, + "loss": 1.07, + "margin_dpo/margin_mean": 190.55233764648438, + "margin_dpo/margin_std": 235.90618896484375, + "step": 286 + }, + { + "KL/chosen_KL_mean": -254.09243774414062, + "KL/mean": -332.7593994140625, + "KL/rejected_KL_mean": -411.42633056640625, + "KL/std": 187.36265563964844, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.002255314029753208, + "fcm_dpo/delta": 0.04675190895795822, + "fcm_dpo/margin": 157.3339080810547, + "fcm_dpo/q_t": 0.41782739758491516, + "grad_norm": 26.561222076416016, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.3640827536582947, + "logits/rejected": -0.3438934087753296, + "logps/chosen": -315.63360595703125, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -489.1224060058594, + "loss": 1.1144, + "margin_dpo/margin_mean": 157.33389282226562, + "margin_dpo/margin_std": 208.7987060546875, + "step": 287 + }, + { + "KL/chosen_KL_mean": -244.06564331054688, + "KL/mean": -340.56097412109375, + "KL/rejected_KL_mean": -437.0563049316406, + "KL/std": 185.27877807617188, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.0022324384190142155, + "fcm_dpo/delta": -0.03332860395312309, + "fcm_dpo/margin": 192.99066162109375, + "fcm_dpo/q_t": 0.39837658405303955, + "grad_norm": 20.748275756835938, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.34643423557281494, + "logits/rejected": -0.3297045826911926, + "logps/chosen": -300.72686767578125, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -524.3920288085938, + "loss": 1.0489, + "margin_dpo/margin_mean": 192.9906768798828, + "margin_dpo/margin_std": 193.25958251953125, + "step": 288 + }, + { + "KL/chosen_KL_mean": -243.8482666015625, + "KL/mean": -350.0272216796875, + "KL/rejected_KL_mean": -456.2061767578125, + "KL/std": 204.0833740234375, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.002224870026111603, + "fcm_dpo/delta": -0.07602076232433319, + "fcm_dpo/margin": 212.35791015625, + "fcm_dpo/q_t": 0.3905183672904968, + "grad_norm": 28.642379760742188, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.32445091009140015, + "logits/rejected": -0.3274286389350891, + "logps/chosen": -289.07867431640625, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -543.848876953125, + "loss": 1.027, + "margin_dpo/margin_mean": 212.35791015625, + "margin_dpo/margin_std": 222.88995361328125, + "step": 289 + }, + { + "KL/chosen_KL_mean": -277.8284912109375, + "KL/mean": -383.560302734375, + "KL/rejected_KL_mean": -489.29205322265625, + "KL/std": 218.40086364746094, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.002185350749641657, + "fcm_dpo/delta": -0.06524636596441269, + "fcm_dpo/margin": 211.4635772705078, + "fcm_dpo/q_t": 0.3948052227497101, + "grad_norm": 30.09307289123535, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.28672564029693604, + "logits/rejected": -0.30779388546943665, + "logps/chosen": -333.29998779296875, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -606.0006103515625, + "loss": 1.0493, + "margin_dpo/margin_mean": 211.46359252929688, + "margin_dpo/margin_std": 259.5584716796875, + "step": 290 + }, + { + "KL/chosen_KL_mean": -247.4573974609375, + "KL/mean": -346.75775146484375, + "KL/rejected_KL_mean": -446.0581359863281, + "KL/std": 195.43624877929688, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.0021787306759506464, + "fcm_dpo/delta": -0.03438626229763031, + "fcm_dpo/margin": 198.60073852539062, + "fcm_dpo/q_t": 0.39908093214035034, + "grad_norm": 22.464242935180664, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.32570880651474, + "logits/rejected": -0.3159620463848114, + "logps/chosen": -310.7177734375, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -535.355224609375, + "loss": 1.0615, + "margin_dpo/margin_mean": 198.60073852539062, + "margin_dpo/margin_std": 231.80142211914062, + "step": 291 + }, + { + "KL/chosen_KL_mean": -273.595703125, + "KL/mean": -379.11517333984375, + "KL/rejected_KL_mean": -484.6346435546875, + "KL/std": 231.96990966796875, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.002155005233362317, + "fcm_dpo/delta": -0.05745099112391472, + "fcm_dpo/margin": 211.0389404296875, + "fcm_dpo/q_t": 0.3955003023147583, + "grad_norm": 22.298507690429688, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.306869775056839, + "logits/rejected": -0.29439347982406616, + "logps/chosen": -327.51422119140625, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -574.5960693359375, + "loss": 1.0622, + "margin_dpo/margin_mean": 211.0389404296875, + "margin_dpo/margin_std": 271.5542907714844, + "step": 292 + }, + { + "KL/chosen_KL_mean": -271.6903076171875, + "KL/mean": -352.81793212890625, + "KL/rejected_KL_mean": -433.945556640625, + "KL/std": 224.78199768066406, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.002160701435059309, + "fcm_dpo/delta": 0.05108872801065445, + "fcm_dpo/margin": 162.25527954101562, + "fcm_dpo/q_t": 0.42054641246795654, + "grad_norm": 43.277870178222656, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.3318144679069519, + "logits/rejected": -0.32798200845718384, + "logps/chosen": -332.06634521484375, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -511.7980041503906, + "loss": 1.1358, + "margin_dpo/margin_mean": 162.25527954101562, + "margin_dpo/margin_std": 253.90650939941406, + "step": 293 + }, + { + "KL/chosen_KL_mean": -250.81201171875, + "KL/mean": -341.93438720703125, + "KL/rejected_KL_mean": -433.05682373046875, + "KL/std": 203.81825256347656, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.002161232754588127, + "fcm_dpo/delta": 0.00632251612842083, + "fcm_dpo/margin": 182.24481201171875, + "fcm_dpo/q_t": 0.4106462299823761, + "grad_norm": 20.052715301513672, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.2955203950405121, + "logits/rejected": -0.3014407753944397, + "logps/chosen": -298.8995361328125, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -514.9537963867188, + "loss": 1.1012, + "margin_dpo/margin_mean": 182.2447967529297, + "margin_dpo/margin_std": 255.03872680664062, + "step": 294 + }, + { + "KL/chosen_KL_mean": -292.969482421875, + "KL/mean": -396.26116943359375, + "KL/rejected_KL_mean": -499.55291748046875, + "KL/std": 229.05368041992188, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.0021535451523959637, + "fcm_dpo/delta": -0.046950262039899826, + "fcm_dpo/margin": 206.58346557617188, + "fcm_dpo/q_t": 0.398703932762146, + "grad_norm": 33.76886749267578, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.3334130048751831, + "logits/rejected": -0.33853596448898315, + "logps/chosen": -342.8941345214844, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -587.0092163085938, + "loss": 1.067, + "margin_dpo/margin_mean": 206.58346557617188, + "margin_dpo/margin_std": 266.01129150390625, + "step": 295 + }, + { + "KL/chosen_KL_mean": -357.1330871582031, + "KL/mean": -428.5017395019531, + "KL/rejected_KL_mean": -499.8703918457031, + "KL/std": 220.69627380371094, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.0021390714682638645, + "fcm_dpo/delta": -0.015223290771245956, + "fcm_dpo/margin": 142.73731994628906, + "fcm_dpo/q_t": 0.42921239137649536, + "grad_norm": 34.340126037597656, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.34358495473861694, + "logits/rejected": -0.32828420400619507, + "logps/chosen": -422.62432861328125, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -592.95947265625, + "loss": 1.187, + "margin_dpo/margin_mean": 142.73731994628906, + "margin_dpo/margin_std": 272.28125, + "step": 296 + }, + { + "KL/chosen_KL_mean": -305.9815673828125, + "KL/mean": -389.3016052246094, + "KL/rejected_KL_mean": -472.62164306640625, + "KL/std": 203.40664672851562, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.0021500743459910154, + "fcm_dpo/delta": 0.04328303784132004, + "fcm_dpo/margin": 166.6400604248047, + "fcm_dpo/q_t": 0.4161558747291565, + "grad_norm": 26.658647537231445, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.3727652132511139, + "logits/rejected": -0.37985509634017944, + "logps/chosen": -362.4585266113281, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -567.7601928710938, + "loss": 1.1066, + "margin_dpo/margin_mean": 166.6400604248047, + "margin_dpo/margin_std": 206.21046447753906, + "step": 297 + }, + { + "KL/chosen_KL_mean": -330.47869873046875, + "KL/mean": -437.3927001953125, + "KL/rejected_KL_mean": -544.3067016601562, + "KL/std": 264.60791015625, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.002145998878404498, + "fcm_dpo/delta": -0.06176210939884186, + "fcm_dpo/margin": 213.82801818847656, + "fcm_dpo/q_t": 0.3990572690963745, + "grad_norm": 24.842294692993164, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.36403989791870117, + "logits/rejected": -0.38525551557540894, + "logps/chosen": -397.8038635253906, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -660.9688720703125, + "loss": 1.0772, + "margin_dpo/margin_mean": 213.82803344726562, + "margin_dpo/margin_std": 310.06292724609375, + "step": 298 + }, + { + "KL/chosen_KL_mean": -274.8798828125, + "KL/mean": -366.14068603515625, + "KL/rejected_KL_mean": -457.4014892578125, + "KL/std": 205.01246643066406, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.002121900673955679, + "fcm_dpo/delta": -0.04373261705040932, + "fcm_dpo/margin": 182.52157592773438, + "fcm_dpo/q_t": 0.4115998148918152, + "grad_norm": 28.827253341674805, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.35734280943870544, + "logits/rejected": -0.36297866702079773, + "logps/chosen": -323.84197998046875, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -541.729736328125, + "loss": 1.1054, + "margin_dpo/margin_mean": 182.52157592773438, + "margin_dpo/margin_std": 253.80422973632812, + "step": 299 + }, + { + "KL/chosen_KL_mean": -340.072265625, + "KL/mean": -449.4986572265625, + "KL/rejected_KL_mean": -558.925048828125, + "KL/std": 240.791748046875, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0020983763970434666, + "fcm_dpo/delta": -0.06205795332789421, + "fcm_dpo/margin": 218.85284423828125, + "fcm_dpo/q_t": 0.3951065242290497, + "grad_norm": 36.48176956176758, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.3916734457015991, + "logits/rejected": -0.3992023766040802, + "logps/chosen": -399.14599609375, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -654.8915405273438, + "loss": 1.0645, + "margin_dpo/margin_mean": 218.85284423828125, + "margin_dpo/margin_std": 286.9918518066406, + "step": 300 + }, + { + "KL/chosen_KL_mean": -285.673583984375, + "KL/mean": -391.4161376953125, + "KL/rejected_KL_mean": -497.1587219238281, + "KL/std": 220.9466552734375, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.00206865556538105, + "fcm_dpo/delta": -0.03946727514266968, + "fcm_dpo/margin": 211.48512268066406, + "fcm_dpo/q_t": 0.40009522438049316, + "grad_norm": 21.42838478088379, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.3892371654510498, + "logits/rejected": -0.39173194766044617, + "logps/chosen": -342.9229736328125, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -589.5122680664062, + "loss": 1.0599, + "margin_dpo/margin_mean": 211.48512268066406, + "margin_dpo/margin_std": 255.37408447265625, + "step": 301 + }, + { + "KL/chosen_KL_mean": -246.42910766601562, + "KL/mean": -337.2757568359375, + "KL/rejected_KL_mean": -428.12237548828125, + "KL/std": 199.16116333007812, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.0020701996982097626, + "fcm_dpo/delta": 0.024608172476291656, + "fcm_dpo/margin": 181.69326782226562, + "fcm_dpo/q_t": 0.4130147695541382, + "grad_norm": 26.431339263916016, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.39726150035858154, + "logits/rejected": -0.4029630422592163, + "logps/chosen": -297.6271057128906, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -525.3487548828125, + "loss": 1.1, + "margin_dpo/margin_mean": 181.69326782226562, + "margin_dpo/margin_std": 231.405517578125, + "step": 302 + }, + { + "KL/chosen_KL_mean": -254.15354919433594, + "KL/mean": -338.04278564453125, + "KL/rejected_KL_mean": -421.9320068359375, + "KL/std": 203.1949005126953, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002091585425660014, + "fcm_dpo/delta": 0.050874415785074234, + "fcm_dpo/margin": 167.77845764160156, + "fcm_dpo/q_t": 0.41896796226501465, + "grad_norm": 26.6970157623291, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.4268595576286316, + "logits/rejected": -0.41521161794662476, + "logps/chosen": -320.86749267578125, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -508.87744140625, + "loss": 1.1277, + "margin_dpo/margin_mean": 167.77847290039062, + "margin_dpo/margin_std": 242.13934326171875, + "step": 303 + }, + { + "KL/chosen_KL_mean": -219.91140747070312, + "KL/mean": -310.89276123046875, + "KL/rejected_KL_mean": -401.87408447265625, + "KL/std": 185.45364379882812, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.002113079186528921, + "fcm_dpo/delta": 0.015879018232226372, + "fcm_dpo/margin": 181.9626922607422, + "fcm_dpo/q_t": 0.4076635539531708, + "grad_norm": 42.516475677490234, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.4222007989883423, + "logits/rejected": -0.41476064920425415, + "logps/chosen": -291.8620910644531, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -492.34613037109375, + "loss": 1.0649, + "margin_dpo/margin_mean": 181.96270751953125, + "margin_dpo/margin_std": 153.8302459716797, + "step": 304 + }, + { + "KL/chosen_KL_mean": -248.52688598632812, + "KL/mean": -331.44976806640625, + "KL/rejected_KL_mean": -414.3726806640625, + "KL/std": 215.2914581298828, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.002095552859827876, + "fcm_dpo/delta": -0.03039616532623768, + "fcm_dpo/margin": 165.8457794189453, + "fcm_dpo/q_t": 0.4201132655143738, + "grad_norm": 24.32717514038086, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.41515517234802246, + "logits/rejected": -0.39973771572113037, + "logps/chosen": -315.3221130371094, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -507.12725830078125, + "loss": 1.1283, + "margin_dpo/margin_mean": 165.8457794189453, + "margin_dpo/margin_std": 236.5407257080078, + "step": 305 + }, + { + "KL/chosen_KL_mean": -245.71923828125, + "KL/mean": -327.5643310546875, + "KL/rejected_KL_mean": -409.4094543457031, + "KL/std": 192.34336853027344, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.0021158938761800528, + "fcm_dpo/delta": 0.05545729026198387, + "fcm_dpo/margin": 163.6902313232422, + "fcm_dpo/q_t": 0.41917920112609863, + "grad_norm": 23.984167098999023, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.41061902046203613, + "logits/rejected": -0.3890068829059601, + "logps/chosen": -315.40313720703125, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -494.56866455078125, + "loss": 1.1134, + "margin_dpo/margin_mean": 163.6902313232422, + "margin_dpo/margin_std": 204.4197998046875, + "step": 306 + }, + { + "KL/chosen_KL_mean": -229.22772216796875, + "KL/mean": -315.544921875, + "KL/rejected_KL_mean": -401.86212158203125, + "KL/std": 181.487548828125, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.0021270960569381714, + "fcm_dpo/delta": 0.03405105322599411, + "fcm_dpo/margin": 172.6343994140625, + "fcm_dpo/q_t": 0.41441789269447327, + "grad_norm": 24.326122283935547, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.38524484634399414, + "logits/rejected": -0.36566612124443054, + "logps/chosen": -299.39312744140625, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -488.8344421386719, + "loss": 1.1048, + "margin_dpo/margin_mean": 172.6343994140625, + "margin_dpo/margin_std": 221.99716186523438, + "step": 307 + }, + { + "KL/chosen_KL_mean": -240.03317260742188, + "KL/mean": -337.109130859375, + "KL/rejected_KL_mean": -434.18505859375, + "KL/std": 206.89242553710938, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.002129509812220931, + "fcm_dpo/delta": -0.014026094228029251, + "fcm_dpo/margin": 194.15188598632812, + "fcm_dpo/q_t": 0.4022546708583832, + "grad_norm": 25.696414947509766, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.36302733421325684, + "logits/rejected": -0.35249167680740356, + "logps/chosen": -295.2781677246094, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -513.5573120117188, + "loss": 1.0585, + "margin_dpo/margin_mean": 194.1519012451172, + "margin_dpo/margin_std": 199.71405029296875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -249.13949584960938, + "KL/mean": -344.7773742675781, + "KL/rejected_KL_mean": -440.41522216796875, + "KL/std": 224.25454711914062, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0021351578179746866, + "fcm_dpo/delta": -0.009223483502864838, + "fcm_dpo/margin": 191.27574157714844, + "fcm_dpo/q_t": 0.40454360842704773, + "grad_norm": 33.72385025024414, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.35267573595046997, + "logits/rejected": -0.34517520666122437, + "logps/chosen": -298.09857177734375, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -522.7559814453125, + "loss": 1.0851, + "margin_dpo/margin_mean": 191.27572631835938, + "margin_dpo/margin_std": 243.2027587890625, + "step": 309 + }, + { + "KL/chosen_KL_mean": -291.47161865234375, + "KL/mean": -384.15106201171875, + "KL/rejected_KL_mean": -476.83050537109375, + "KL/std": 204.56735229492188, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.0021327342838048935, + "fcm_dpo/delta": 0.004583075642585754, + "fcm_dpo/margin": 185.35890197753906, + "fcm_dpo/q_t": 0.40686386823654175, + "grad_norm": 32.27021026611328, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.3865277171134949, + "logits/rejected": -0.3740183413028717, + "logps/chosen": -354.2134094238281, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -556.7608032226562, + "loss": 1.0748, + "margin_dpo/margin_mean": 185.35888671875, + "margin_dpo/margin_std": 200.1402587890625, + "step": 310 + }, + { + "KL/chosen_KL_mean": -307.4329833984375, + "KL/mean": -409.7182922363281, + "KL/rejected_KL_mean": -512.0036010742188, + "KL/std": 255.78775024414062, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.0021153478883206844, + "fcm_dpo/delta": -0.034209851175546646, + "fcm_dpo/margin": 204.57061767578125, + "fcm_dpo/q_t": 0.4021032154560089, + "grad_norm": 27.51216697692871, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.3856719434261322, + "logits/rejected": -0.3630024194717407, + "logps/chosen": -360.4609680175781, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -589.4417724609375, + "loss": 1.0747, + "margin_dpo/margin_mean": 204.57061767578125, + "margin_dpo/margin_std": 270.60272216796875, + "step": 311 + }, + { + "KL/chosen_KL_mean": -296.1390380859375, + "KL/mean": -390.2773742675781, + "KL/rejected_KL_mean": -484.415771484375, + "KL/std": 233.6208038330078, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.0021023908630013466, + "fcm_dpo/delta": -0.04076831415295601, + "fcm_dpo/margin": 188.27670288085938, + "fcm_dpo/q_t": 0.4114304780960083, + "grad_norm": 30.539905548095703, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.3101688027381897, + "logits/rejected": -0.29841965436935425, + "logps/chosen": -345.53125, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -560.2085571289062, + "loss": 1.1058, + "margin_dpo/margin_mean": 188.27670288085938, + "margin_dpo/margin_std": 277.10235595703125, + "step": 312 + }, + { + "KL/chosen_KL_mean": -271.54583740234375, + "KL/mean": -370.45465087890625, + "KL/rejected_KL_mean": -469.36346435546875, + "KL/std": 238.268310546875, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.002077316865324974, + "fcm_dpo/delta": -0.07944377511739731, + "fcm_dpo/margin": 197.81759643554688, + "fcm_dpo/q_t": 0.4055682420730591, + "grad_norm": 21.716537475585938, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.40294545888900757, + "logits/rejected": -0.40182724595069885, + "logps/chosen": -321.6985778808594, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -555.7696533203125, + "loss": 1.0923, + "margin_dpo/margin_mean": 197.81759643554688, + "margin_dpo/margin_std": 266.08221435546875, + "step": 313 + }, + { + "KL/chosen_KL_mean": -290.51409912109375, + "KL/mean": -381.05548095703125, + "KL/rejected_KL_mean": -471.596923828125, + "KL/std": 229.35504150390625, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0020488370209932327, + "fcm_dpo/delta": -0.029144132509827614, + "fcm_dpo/margin": 181.08282470703125, + "fcm_dpo/q_t": 0.41715776920318604, + "grad_norm": 20.128904342651367, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.45148593187332153, + "logits/rejected": -0.4533792734146118, + "logps/chosen": -347.75164794921875, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -569.1934814453125, + "loss": 1.1337, + "margin_dpo/margin_mean": 181.08282470703125, + "margin_dpo/margin_std": 293.7008972167969, + "step": 314 + }, + { + "KL/chosen_KL_mean": -255.66592407226562, + "KL/mean": -340.78509521484375, + "KL/rejected_KL_mean": -425.90423583984375, + "KL/std": 203.66940307617188, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.002061174251139164, + "fcm_dpo/delta": 0.05092431977391243, + "fcm_dpo/margin": 170.23834228515625, + "fcm_dpo/q_t": 0.4188379943370819, + "grad_norm": 19.817676544189453, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.35196053981781006, + "logits/rejected": -0.35638299584388733, + "logps/chosen": -304.7355041503906, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -511.58514404296875, + "loss": 1.1177, + "margin_dpo/margin_mean": 170.2383575439453, + "margin_dpo/margin_std": 227.2279052734375, + "step": 315 + }, + { + "KL/chosen_KL_mean": -255.556640625, + "KL/mean": -373.936767578125, + "KL/rejected_KL_mean": -492.3169250488281, + "KL/std": 234.8746337890625, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0020339572802186012, + "fcm_dpo/delta": -0.08651921153068542, + "fcm_dpo/margin": 236.76031494140625, + "fcm_dpo/q_t": 0.3899900019168854, + "grad_norm": 22.462806701660156, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.3865298926830292, + "logits/rejected": -0.39097899198532104, + "logps/chosen": -309.8173828125, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -593.598388671875, + "loss": 1.0263, + "margin_dpo/margin_mean": 236.76031494140625, + "margin_dpo/margin_std": 259.88873291015625, + "step": 316 + }, + { + "KL/chosen_KL_mean": -265.3761291503906, + "KL/mean": -394.6961669921875, + "KL/rejected_KL_mean": -524.01611328125, + "KL/std": 216.79380798339844, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.002005034126341343, + "fcm_dpo/delta": -0.12489780783653259, + "fcm_dpo/margin": 258.64007568359375, + "fcm_dpo/q_t": 0.37905311584472656, + "grad_norm": 29.764869689941406, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.39593786001205444, + "logits/rejected": -0.38730502128601074, + "logps/chosen": -321.4703369140625, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -624.7152099609375, + "loss": 0.9921, + "margin_dpo/margin_mean": 258.64007568359375, + "margin_dpo/margin_std": 243.14756774902344, + "step": 317 + }, + { + "KL/chosen_KL_mean": -291.64898681640625, + "KL/mean": -388.96954345703125, + "KL/rejected_KL_mean": -486.2900390625, + "KL/std": 229.6132354736328, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0019934140145778656, + "fcm_dpo/delta": 0.012284556403756142, + "fcm_dpo/margin": 194.64102172851562, + "fcm_dpo/q_t": 0.4105232357978821, + "grad_norm": 22.980838775634766, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.4225706458091736, + "logits/rejected": -0.393215537071228, + "logps/chosen": -356.2947082519531, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -569.0543212890625, + "loss": 1.0926, + "margin_dpo/margin_mean": 194.64102172851562, + "margin_dpo/margin_std": 248.1627197265625, + "step": 318 + }, + { + "KL/chosen_KL_mean": -263.8323669433594, + "KL/mean": -380.08062744140625, + "KL/rejected_KL_mean": -496.32891845703125, + "KL/std": 247.440185546875, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.001971443183720112, + "fcm_dpo/delta": -0.06114393472671509, + "fcm_dpo/margin": 232.49655151367188, + "fcm_dpo/q_t": 0.393601655960083, + "grad_norm": 22.2618350982666, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.4032759368419647, + "logits/rejected": -0.41747260093688965, + "logps/chosen": -313.21612548828125, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -610.2354125976562, + "loss": 1.0435, + "margin_dpo/margin_mean": 232.49655151367188, + "margin_dpo/margin_std": 266.54632568359375, + "step": 319 + }, + { + "KL/chosen_KL_mean": -286.0980224609375, + "KL/mean": -413.4766540527344, + "KL/rejected_KL_mean": -540.8552856445312, + "KL/std": 262.2138366699219, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.0019302340224385262, + "fcm_dpo/delta": -0.0970359742641449, + "fcm_dpo/margin": 254.75723266601562, + "fcm_dpo/q_t": 0.3870835304260254, + "grad_norm": 26.517669677734375, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.41013917326927185, + "logits/rejected": -0.4170839488506317, + "logps/chosen": -345.6029052734375, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -638.5224609375, + "loss": 1.0205, + "margin_dpo/margin_mean": 254.75723266601562, + "margin_dpo/margin_std": 279.715576171875, + "step": 320 + }, + { + "KL/chosen_KL_mean": -352.2861328125, + "KL/mean": -459.5243225097656, + "KL/rejected_KL_mean": -566.762451171875, + "KL/std": 262.43017578125, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.001913099200464785, + "fcm_dpo/delta": -0.011455025523900986, + "fcm_dpo/margin": 214.47637939453125, + "fcm_dpo/q_t": 0.4060276746749878, + "grad_norm": 26.823516845703125, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.34034955501556396, + "logits/rejected": -0.32983988523483276, + "logps/chosen": -413.8348083496094, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -658.403564453125, + "loss": 1.0874, + "margin_dpo/margin_mean": 214.47637939453125, + "margin_dpo/margin_std": 280.22088623046875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -304.80242919921875, + "KL/mean": -422.7059326171875, + "KL/rejected_KL_mean": -540.609375, + "KL/std": 238.2976531982422, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.001901952549815178, + "fcm_dpo/delta": -0.051145751029253006, + "fcm_dpo/margin": 235.80699157714844, + "fcm_dpo/q_t": 0.3951281011104584, + "grad_norm": 21.14191436767578, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.35637927055358887, + "logits/rejected": -0.35669732093811035, + "logps/chosen": -362.09246826171875, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -636.359375, + "loss": 1.0418, + "margin_dpo/margin_mean": 235.80699157714844, + "margin_dpo/margin_std": 250.65390014648438, + "step": 322 + }, + { + "KL/chosen_KL_mean": -334.8727722167969, + "KL/mean": -434.36474609375, + "KL/rejected_KL_mean": -533.856689453125, + "KL/std": 238.89381408691406, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.0019058396574109793, + "fcm_dpo/delta": 0.021561458706855774, + "fcm_dpo/margin": 198.98394775390625, + "fcm_dpo/q_t": 0.4116159975528717, + "grad_norm": 36.634178161621094, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.3893946707248688, + "logits/rejected": -0.3900691270828247, + "logps/chosen": -386.10675048828125, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -608.9186401367188, + "loss": 1.1019, + "margin_dpo/margin_mean": 198.98394775390625, + "margin_dpo/margin_std": 256.5784912109375, + "step": 323 + }, + { + "KL/chosen_KL_mean": -380.1344909667969, + "KL/mean": -474.47552490234375, + "KL/rejected_KL_mean": -568.8165893554688, + "KL/std": 256.23291015625, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.001901252195239067, + "fcm_dpo/delta": -0.05288267135620117, + "fcm_dpo/margin": 188.68209838867188, + "fcm_dpo/q_t": 0.4187321364879608, + "grad_norm": 42.7255973815918, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.3902486562728882, + "logits/rejected": -0.37962621450424194, + "logps/chosen": -445.2696533203125, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -655.2940673828125, + "loss": 1.1385, + "margin_dpo/margin_mean": 188.68209838867188, + "margin_dpo/margin_std": 290.64349365234375, + "step": 324 + }, + { + "KL/chosen_KL_mean": -291.7728576660156, + "KL/mean": -376.75042724609375, + "KL/rejected_KL_mean": -461.72802734375, + "KL/std": 225.8763885498047, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.001914022141136229, + "fcm_dpo/delta": 0.07717220485210419, + "fcm_dpo/margin": 169.95516967773438, + "fcm_dpo/q_t": 0.42427897453308105, + "grad_norm": 24.78872299194336, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.4337829053401947, + "logits/rejected": -0.41769248247146606, + "logps/chosen": -347.98846435546875, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -531.81396484375, + "loss": 1.1373, + "margin_dpo/margin_mean": 169.95516967773438, + "margin_dpo/margin_std": 231.92037963867188, + "step": 325 + }, + { + "KL/chosen_KL_mean": -327.98687744140625, + "KL/mean": -410.3250427246094, + "KL/rejected_KL_mean": -492.6632080078125, + "KL/std": 212.76693725585938, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.001930159516632557, + "fcm_dpo/delta": 0.024788817390799522, + "fcm_dpo/margin": 164.67636108398438, + "fcm_dpo/q_t": 0.42440783977508545, + "grad_norm": 45.59063720703125, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.44264641404151917, + "logits/rejected": -0.42518919706344604, + "logps/chosen": -400.7117919921875, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -572.510009765625, + "loss": 1.1326, + "margin_dpo/margin_mean": 164.67636108398438, + "margin_dpo/margin_std": 205.204345703125, + "step": 326 + }, + { + "KL/chosen_KL_mean": -292.86968994140625, + "KL/mean": -412.77789306640625, + "KL/rejected_KL_mean": -532.6860961914062, + "KL/std": 228.66757202148438, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0019071187125518918, + "fcm_dpo/delta": -0.06047464907169342, + "fcm_dpo/margin": 239.81637573242188, + "fcm_dpo/q_t": 0.392434298992157, + "grad_norm": 36.25203323364258, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.4332086741924286, + "logits/rejected": -0.4247819781303406, + "logps/chosen": -362.0041198730469, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -644.619873046875, + "loss": 1.0357, + "margin_dpo/margin_mean": 239.81637573242188, + "margin_dpo/margin_std": 252.48092651367188, + "step": 327 + }, + { + "KL/chosen_KL_mean": -298.658203125, + "KL/mean": -406.617431640625, + "KL/rejected_KL_mean": -514.5765991210938, + "KL/std": 249.38577270507812, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.00190459075383842, + "fcm_dpo/delta": -0.011717304587364197, + "fcm_dpo/margin": 215.91836547851562, + "fcm_dpo/q_t": 0.4054545760154724, + "grad_norm": 24.519594192504883, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.41552045941352844, + "logits/rejected": -0.41982895135879517, + "logps/chosen": -358.34539794921875, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -605.431640625, + "loss": 1.079, + "margin_dpo/margin_mean": 215.9183807373047, + "margin_dpo/margin_std": 272.96502685546875, + "step": 328 + }, + { + "KL/chosen_KL_mean": -335.6679382324219, + "KL/mean": -452.0716552734375, + "KL/rejected_KL_mean": -568.475341796875, + "KL/std": 266.67779541015625, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.0018819253891706467, + "fcm_dpo/delta": -0.04054764658212662, + "fcm_dpo/margin": 232.80743408203125, + "fcm_dpo/q_t": 0.39829006791114807, + "grad_norm": 23.63628578186035, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.45960137248039246, + "logits/rejected": -0.4569237232208252, + "logps/chosen": -400.91412353515625, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -669.173095703125, + "loss": 1.0642, + "margin_dpo/margin_mean": 232.80743408203125, + "margin_dpo/margin_std": 280.071533203125, + "step": 329 + }, + { + "KL/chosen_KL_mean": -281.56884765625, + "KL/mean": -398.327392578125, + "KL/rejected_KL_mean": -515.0859375, + "KL/std": 250.85325622558594, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0018758606165647507, + "fcm_dpo/delta": -0.03979107737541199, + "fcm_dpo/margin": 233.51708984375, + "fcm_dpo/q_t": 0.398856520652771, + "grad_norm": 28.41822052001953, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.42625749111175537, + "logits/rejected": -0.42764711380004883, + "logps/chosen": -328.56719970703125, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -601.9627685546875, + "loss": 1.0573, + "margin_dpo/margin_mean": 233.51708984375, + "margin_dpo/margin_std": 275.60577392578125, + "step": 330 + }, + { + "KL/chosen_KL_mean": -305.76800537109375, + "KL/mean": -424.60296630859375, + "KL/rejected_KL_mean": -543.43798828125, + "KL/std": 209.80612182617188, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.0018607673700898886, + "fcm_dpo/delta": -0.04418721795082092, + "fcm_dpo/margin": 237.66995239257812, + "fcm_dpo/q_t": 0.3949892520904541, + "grad_norm": 27.10124397277832, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.4729953110218048, + "logits/rejected": -0.47148242592811584, + "logps/chosen": -356.292236328125, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -632.453369140625, + "loss": 1.027, + "margin_dpo/margin_mean": 237.66995239257812, + "margin_dpo/margin_std": 208.02056884765625, + "step": 331 + }, + { + "KL/chosen_KL_mean": -309.21405029296875, + "KL/mean": -394.34515380859375, + "KL/rejected_KL_mean": -479.4761962890625, + "KL/std": 222.29739379882812, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.001860418589785695, + "fcm_dpo/delta": 0.025393592193722725, + "fcm_dpo/margin": 170.2621612548828, + "fcm_dpo/q_t": 0.42723387479782104, + "grad_norm": 23.805442810058594, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.4660256505012512, + "logits/rejected": -0.445956826210022, + "logps/chosen": -358.39434814453125, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -555.9613647460938, + "loss": 1.1452, + "margin_dpo/margin_mean": 170.26214599609375, + "margin_dpo/margin_std": 252.012451171875, + "step": 332 + }, + { + "KL/chosen_KL_mean": -341.1619873046875, + "KL/mean": -455.632080078125, + "KL/rejected_KL_mean": -570.1022338867188, + "KL/std": 261.497314453125, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.001838655211031437, + "fcm_dpo/delta": -0.07151585817337036, + "fcm_dpo/margin": 228.9402618408203, + "fcm_dpo/q_t": 0.4039188027381897, + "grad_norm": 29.533124923706055, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.4914425015449524, + "logits/rejected": -0.4861595928668976, + "logps/chosen": -404.917724609375, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -665.1463623046875, + "loss": 1.0911, + "margin_dpo/margin_mean": 228.94024658203125, + "margin_dpo/margin_std": 317.145263671875, + "step": 333 + }, + { + "KL/chosen_KL_mean": -335.464111328125, + "KL/mean": -436.04254150390625, + "KL/rejected_KL_mean": -536.6209716796875, + "KL/std": 284.3128662109375, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0018430267227813601, + "fcm_dpo/delta": 0.030359743162989616, + "fcm_dpo/margin": 201.15689086914062, + "fcm_dpo/q_t": 0.4161614775657654, + "grad_norm": 28.042444229125977, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.5175400376319885, + "logits/rejected": -0.5172750353813171, + "logps/chosen": -402.4438781738281, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -631.9379272460938, + "loss": 1.1326, + "margin_dpo/margin_mean": 201.15687561035156, + "margin_dpo/margin_std": 320.84222412109375, + "step": 334 + }, + { + "KL/chosen_KL_mean": -361.9266357421875, + "KL/mean": -444.96441650390625, + "KL/rejected_KL_mean": -528.002197265625, + "KL/std": 275.4105224609375, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.0018500408623367548, + "fcm_dpo/delta": 0.001012663939036429, + "fcm_dpo/margin": 166.0755615234375, + "fcm_dpo/q_t": 0.43012675642967224, + "grad_norm": 33.411529541015625, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.5440207123756409, + "logits/rejected": -0.5235892534255981, + "logps/chosen": -424.4691162109375, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -615.619873046875, + "loss": 1.1737, + "margin_dpo/margin_mean": 166.0755615234375, + "margin_dpo/margin_std": 293.3839111328125, + "step": 335 + }, + { + "KL/chosen_KL_mean": -369.8343811035156, + "KL/mean": -495.605712890625, + "KL/rejected_KL_mean": -621.3770751953125, + "KL/std": 308.52398681640625, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.001839759061113, + "fcm_dpo/delta": -0.06589866429567337, + "fcm_dpo/margin": 251.54266357421875, + "fcm_dpo/q_t": 0.39720821380615234, + "grad_norm": 32.32990264892578, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.48386096954345703, + "logits/rejected": -0.48270291090011597, + "logps/chosen": -424.36553955078125, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -725.78125, + "loss": 1.08, + "margin_dpo/margin_mean": 251.54266357421875, + "margin_dpo/margin_std": 361.5411376953125, + "step": 336 + }, + { + "KL/chosen_KL_mean": -319.493896484375, + "KL/mean": -444.7812194824219, + "KL/rejected_KL_mean": -570.0685424804688, + "KL/std": 244.06655883789062, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.0018074819818139076, + "fcm_dpo/delta": -0.05560125783085823, + "fcm_dpo/margin": 250.5746612548828, + "fcm_dpo/q_t": 0.3959052562713623, + "grad_norm": 27.28701400756836, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.5211169719696045, + "logits/rejected": -0.5096985697746277, + "logps/chosen": -384.62255859375, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -671.7955322265625, + "loss": 1.0477, + "margin_dpo/margin_mean": 250.5746612548828, + "margin_dpo/margin_std": 292.10760498046875, + "step": 337 + }, + { + "KL/chosen_KL_mean": -289.342041015625, + "KL/mean": -410.1605529785156, + "KL/rejected_KL_mean": -530.9790649414062, + "KL/std": 239.140625, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.0017946097068488598, + "fcm_dpo/delta": -0.035223063081502914, + "fcm_dpo/margin": 241.63694763183594, + "fcm_dpo/q_t": 0.3981233537197113, + "grad_norm": 28.0240421295166, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.5246094465255737, + "logits/rejected": -0.5211294889450073, + "logps/chosen": -347.7647705078125, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -620.047607421875, + "loss": 1.0447, + "margin_dpo/margin_mean": 241.636962890625, + "margin_dpo/margin_std": 246.10772705078125, + "step": 338 + }, + { + "KL/chosen_KL_mean": -322.6174011230469, + "KL/mean": -438.80010986328125, + "KL/rejected_KL_mean": -554.9828491210938, + "KL/std": 249.94732666015625, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.0017756065353751183, + "fcm_dpo/delta": -0.09871108829975128, + "fcm_dpo/margin": 232.36544799804688, + "fcm_dpo/q_t": 0.403381884098053, + "grad_norm": 30.086326599121094, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.5407413244247437, + "logits/rejected": -0.5262603759765625, + "logps/chosen": -382.61273193359375, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -658.8937377929688, + "loss": 1.0723, + "margin_dpo/margin_mean": 232.36544799804688, + "margin_dpo/margin_std": 254.37350463867188, + "step": 339 + }, + { + "KL/chosen_KL_mean": -334.1612854003906, + "KL/mean": -438.4512939453125, + "KL/rejected_KL_mean": -542.7413330078125, + "KL/std": 242.02891540527344, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0017510585021227598, + "fcm_dpo/delta": -0.020008647814393044, + "fcm_dpo/margin": 208.5800018310547, + "fcm_dpo/q_t": 0.4155506491661072, + "grad_norm": 30.712139129638672, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.4409928619861603, + "logits/rejected": -0.4209999442100525, + "logps/chosen": -386.99151611328125, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -615.8485107421875, + "loss": 1.1176, + "margin_dpo/margin_mean": 208.5800018310547, + "margin_dpo/margin_std": 294.775390625, + "step": 340 + }, + { + "KL/chosen_KL_mean": -324.33526611328125, + "KL/mean": -443.5288391113281, + "KL/rejected_KL_mean": -562.722412109375, + "KL/std": 255.23745727539062, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0017410940490663052, + "fcm_dpo/delta": -0.015985984355211258, + "fcm_dpo/margin": 238.38714599609375, + "fcm_dpo/q_t": 0.4035840630531311, + "grad_norm": 24.19179916381836, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.4436286687850952, + "logits/rejected": -0.4443032741546631, + "logps/chosen": -372.23504638671875, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -664.5322875976562, + "loss": 1.0681, + "margin_dpo/margin_mean": 238.38714599609375, + "margin_dpo/margin_std": 270.8433837890625, + "step": 341 + }, + { + "KL/chosen_KL_mean": -321.30511474609375, + "KL/mean": -425.1126708984375, + "KL/rejected_KL_mean": -528.9202880859375, + "KL/std": 244.9583740234375, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.001742843072861433, + "fcm_dpo/delta": -0.015066071413457394, + "fcm_dpo/margin": 207.61514282226562, + "fcm_dpo/q_t": 0.41575270891189575, + "grad_norm": 34.45230484008789, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.4663882255554199, + "logits/rejected": -0.440545916557312, + "logps/chosen": -393.3017578125, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -621.5098876953125, + "loss": 1.1168, + "margin_dpo/margin_mean": 207.6151580810547, + "margin_dpo/margin_std": 289.6087646484375, + "step": 342 + }, + { + "KL/chosen_KL_mean": -295.3658447265625, + "KL/mean": -418.901611328125, + "KL/rejected_KL_mean": -542.4373779296875, + "KL/std": 239.45431518554688, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.0017327230889350176, + "fcm_dpo/delta": -0.029362058266997337, + "fcm_dpo/margin": 247.071533203125, + "fcm_dpo/q_t": 0.39798176288604736, + "grad_norm": 23.32623291015625, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.47054433822631836, + "logits/rejected": -0.4736904501914978, + "logps/chosen": -349.7714538574219, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -653.478759765625, + "loss": 1.0366, + "margin_dpo/margin_mean": 247.071533203125, + "margin_dpo/margin_std": 211.33450317382812, + "step": 343 + }, + { + "KL/chosen_KL_mean": -287.8268737792969, + "KL/mean": -400.8477783203125, + "KL/rejected_KL_mean": -513.8687133789062, + "KL/std": 272.7809143066406, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.001715958584100008, + "fcm_dpo/delta": -0.08199367672204971, + "fcm_dpo/margin": 226.0418701171875, + "fcm_dpo/q_t": 0.4118735194206238, + "grad_norm": 30.284103393554688, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.4321858882904053, + "logits/rejected": -0.43335849046707153, + "logps/chosen": -341.7915344238281, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -604.4920654296875, + "loss": 1.096, + "margin_dpo/margin_mean": 226.0418701171875, + "margin_dpo/margin_std": 281.4267578125, + "step": 344 + }, + { + "KL/chosen_KL_mean": -338.961669921875, + "KL/mean": -462.3531799316406, + "KL/rejected_KL_mean": -585.7446899414062, + "KL/std": 249.0282440185547, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0017002711538225412, + "fcm_dpo/delta": -0.020503249019384384, + "fcm_dpo/margin": 246.7830047607422, + "fcm_dpo/q_t": 0.4016736149787903, + "grad_norm": 26.957731246948242, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.4378073811531067, + "logits/rejected": -0.4355306923389435, + "logps/chosen": -400.6473693847656, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -685.235107421875, + "loss": 1.06, + "margin_dpo/margin_mean": 246.7830047607422, + "margin_dpo/margin_std": 270.58734130859375, + "step": 345 + }, + { + "KL/chosen_KL_mean": -339.72650146484375, + "KL/mean": -453.78973388671875, + "KL/rejected_KL_mean": -567.8529663085938, + "KL/std": 250.03469848632812, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.001697094994597137, + "fcm_dpo/delta": 0.013364390470087528, + "fcm_dpo/margin": 228.1264190673828, + "fcm_dpo/q_t": 0.4083371162414551, + "grad_norm": 27.32826805114746, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.42807716131210327, + "logits/rejected": -0.4175646901130676, + "logps/chosen": -394.9827575683594, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -645.268310546875, + "loss": 1.0841, + "margin_dpo/margin_mean": 228.1264190673828, + "margin_dpo/margin_std": 264.008544921875, + "step": 346 + }, + { + "KL/chosen_KL_mean": -346.0872802734375, + "KL/mean": -459.3369140625, + "KL/rejected_KL_mean": -572.5865478515625, + "KL/std": 264.540771484375, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.0017041533719748259, + "fcm_dpo/delta": 0.014557472430169582, + "fcm_dpo/margin": 226.49923706054688, + "fcm_dpo/q_t": 0.41066300868988037, + "grad_norm": 22.713058471679688, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.4242118299007416, + "logits/rejected": -0.41930991411209106, + "logps/chosen": -403.65350341796875, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -664.941650390625, + "loss": 1.0965, + "margin_dpo/margin_mean": 226.49923706054688, + "margin_dpo/margin_std": 295.05206298828125, + "step": 347 + }, + { + "KL/chosen_KL_mean": -310.332763671875, + "KL/mean": -408.4686279296875, + "KL/rejected_KL_mean": -506.6045227050781, + "KL/std": 229.6025390625, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.0016961813671514392, + "fcm_dpo/delta": -0.02421555668115616, + "fcm_dpo/margin": 196.27175903320312, + "fcm_dpo/q_t": 0.4223266839981079, + "grad_norm": 24.695621490478516, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.4083217978477478, + "logits/rejected": -0.4073137938976288, + "logps/chosen": -366.65045166015625, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -595.742919921875, + "loss": 1.1307, + "margin_dpo/margin_mean": 196.271728515625, + "margin_dpo/margin_std": 263.52020263671875, + "step": 348 + }, + { + "KL/chosen_KL_mean": -301.2662353515625, + "KL/mean": -414.44342041015625, + "KL/rejected_KL_mean": -527.62060546875, + "KL/std": 259.91021728515625, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.0017063779523596168, + "fcm_dpo/delta": 0.013939771801233292, + "fcm_dpo/margin": 226.3544158935547, + "fcm_dpo/q_t": 0.41211697459220886, + "grad_norm": 20.769058227539062, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.42274969816207886, + "logits/rejected": -0.4229254126548767, + "logps/chosen": -359.291748046875, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -625.125732421875, + "loss": 1.1069, + "margin_dpo/margin_mean": 226.35440063476562, + "margin_dpo/margin_std": 318.92498779296875, + "step": 349 + }, + { + "KL/chosen_KL_mean": -333.0226745605469, + "KL/mean": -440.72711181640625, + "KL/rejected_KL_mean": -548.4315185546875, + "KL/std": 251.5546875, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.0016938135959208012, + "fcm_dpo/delta": -0.02092406153678894, + "fcm_dpo/margin": 215.40890502929688, + "fcm_dpo/q_t": 0.4150155186653137, + "grad_norm": 30.646764755249023, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.4468514323234558, + "logits/rejected": -0.45373594760894775, + "logps/chosen": -397.3531494140625, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -638.30322265625, + "loss": 1.1229, + "margin_dpo/margin_mean": 215.40892028808594, + "margin_dpo/margin_std": 316.351806640625, + "step": 350 + }, + { + "KL/chosen_KL_mean": -306.000244140625, + "KL/mean": -440.10308837890625, + "KL/rejected_KL_mean": -574.2059936523438, + "KL/std": 294.214599609375, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0016751708462834358, + "fcm_dpo/delta": -0.05200861394405365, + "fcm_dpo/margin": 268.20574951171875, + "fcm_dpo/q_t": 0.3986712694168091, + "grad_norm": 28.40315818786621, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.42266491055488586, + "logits/rejected": -0.4223342835903168, + "logps/chosen": -366.67236328125, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -675.7714233398438, + "loss": 1.0628, + "margin_dpo/margin_mean": 268.20574951171875, + "margin_dpo/margin_std": 342.70318603515625, + "step": 351 + }, + { + "KL/chosen_KL_mean": -363.4024658203125, + "KL/mean": -444.3284912109375, + "KL/rejected_KL_mean": -525.2545166015625, + "KL/std": 270.6835021972656, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.001686844159848988, + "fcm_dpo/delta": 0.03139277547597885, + "fcm_dpo/margin": 161.85205078125, + "fcm_dpo/q_t": 0.4369640350341797, + "grad_norm": 35.98271942138672, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.44371330738067627, + "logits/rejected": -0.4154715836048126, + "logps/chosen": -434.3459167480469, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -601.896484375, + "loss": 1.1974, + "margin_dpo/margin_mean": 161.85205078125, + "margin_dpo/margin_std": 307.3551025390625, + "step": 352 + }, + { + "KL/chosen_KL_mean": -338.1783447265625, + "KL/mean": -447.04248046875, + "KL/rejected_KL_mean": -555.9066162109375, + "KL/std": 267.84136962890625, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.0016746262554079294, + "fcm_dpo/delta": -0.03634690120816231, + "fcm_dpo/margin": 217.72824096679688, + "fcm_dpo/q_t": 0.41584479808807373, + "grad_norm": 20.67302131652832, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.4561423063278198, + "logits/rejected": -0.4470541477203369, + "logps/chosen": -396.5736999511719, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -636.2421264648438, + "loss": 1.1189, + "margin_dpo/margin_mean": 217.72824096679688, + "margin_dpo/margin_std": 306.3041076660156, + "step": 353 + }, + { + "KL/chosen_KL_mean": -275.72650146484375, + "KL/mean": -396.62957763671875, + "KL/rejected_KL_mean": -517.5325927734375, + "KL/std": 245.72369384765625, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.0016714767552912235, + "fcm_dpo/delta": -0.004385399632155895, + "fcm_dpo/margin": 241.80613708496094, + "fcm_dpo/q_t": 0.4048001170158386, + "grad_norm": 31.205543518066406, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.4049034118652344, + "logits/rejected": -0.39788562059402466, + "logps/chosen": -335.52947998046875, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -606.2901611328125, + "loss": 1.0628, + "margin_dpo/margin_mean": 241.80613708496094, + "margin_dpo/margin_std": 246.92401123046875, + "step": 354 + }, + { + "KL/chosen_KL_mean": -279.00933837890625, + "KL/mean": -394.15985107421875, + "KL/rejected_KL_mean": -509.3103332519531, + "KL/std": 240.28662109375, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.001674377708695829, + "fcm_dpo/delta": 0.01494278572499752, + "fcm_dpo/margin": 230.30096435546875, + "fcm_dpo/q_t": 0.4084288775920868, + "grad_norm": 36.49629211425781, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.4683745205402374, + "logits/rejected": -0.4626787006855011, + "logps/chosen": -333.1378479003906, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -591.7164306640625, + "loss": 1.0759, + "margin_dpo/margin_mean": 230.3009796142578, + "margin_dpo/margin_std": 238.81350708007812, + "step": 355 + }, + { + "KL/chosen_KL_mean": -345.0101318359375, + "KL/mean": -409.93914794921875, + "KL/rejected_KL_mean": -474.8681945800781, + "KL/std": 262.22021484375, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.0016855915309861302, + "fcm_dpo/delta": 0.044053323566913605, + "fcm_dpo/margin": 129.8580322265625, + "fcm_dpo/q_t": 0.45041394233703613, + "grad_norm": 36.29426574707031, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.4423523545265198, + "logits/rejected": -0.42212456464767456, + "logps/chosen": -409.6839599609375, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -550.7674560546875, + "loss": 1.2518, + "margin_dpo/margin_mean": 129.8580322265625, + "margin_dpo/margin_std": 324.0181884765625, + "step": 356 + }, + { + "KL/chosen_KL_mean": -306.63751220703125, + "KL/mean": -412.8591003417969, + "KL/rejected_KL_mean": -519.0806884765625, + "KL/std": 252.15200805664062, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.001686917501501739, + "fcm_dpo/delta": -0.018046293407678604, + "fcm_dpo/margin": 212.44320678710938, + "fcm_dpo/q_t": 0.41526633501052856, + "grad_norm": 24.757951736450195, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.46244481205940247, + "logits/rejected": -0.4534793496131897, + "logps/chosen": -359.36328125, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -605.921875, + "loss": 1.1073, + "margin_dpo/margin_mean": 212.44322204589844, + "margin_dpo/margin_std": 261.8765869140625, + "step": 357 + }, + { + "KL/chosen_KL_mean": -283.7861022949219, + "KL/mean": -382.4645690917969, + "KL/rejected_KL_mean": -481.1430358886719, + "KL/std": 245.43194580078125, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.001703993882983923, + "fcm_dpo/delta": 0.06590504944324493, + "fcm_dpo/margin": 197.35694885253906, + "fcm_dpo/q_t": 0.4224773049354553, + "grad_norm": 26.503250122070312, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.4892391562461853, + "logits/rejected": -0.47518491744995117, + "logps/chosen": -346.9915466308594, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -569.516357421875, + "loss": 1.1285, + "margin_dpo/margin_mean": 197.35694885253906, + "margin_dpo/margin_std": 271.3324890136719, + "step": 358 + }, + { + "KL/chosen_KL_mean": -321.45391845703125, + "KL/mean": -435.9278564453125, + "KL/rejected_KL_mean": -550.4017944335938, + "KL/std": 247.4920654296875, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.001709858188405633, + "fcm_dpo/delta": 0.008867787197232246, + "fcm_dpo/margin": 228.9478759765625, + "fcm_dpo/q_t": 0.4099605083465576, + "grad_norm": 32.5715446472168, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.4548218250274658, + "logits/rejected": -0.44962620735168457, + "logps/chosen": -377.82415771484375, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -632.5755615234375, + "loss": 1.0947, + "margin_dpo/margin_mean": 228.9478759765625, + "margin_dpo/margin_std": 296.9607849121094, + "step": 359 + }, + { + "KL/chosen_KL_mean": -316.53680419921875, + "KL/mean": -407.337890625, + "KL/rejected_KL_mean": -498.13897705078125, + "KL/std": 209.03961181640625, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.00171757978387177, + "fcm_dpo/delta": 0.03054092638194561, + "fcm_dpo/margin": 181.60220336914062, + "fcm_dpo/q_t": 0.4255198538303375, + "grad_norm": 37.421810150146484, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.46587732434272766, + "logits/rejected": -0.45288413763046265, + "logps/chosen": -367.9971923828125, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -567.9779052734375, + "loss": 1.129, + "margin_dpo/margin_mean": 181.60220336914062, + "margin_dpo/margin_std": 206.04116821289062, + "step": 360 + }, + { + "KL/chosen_KL_mean": -339.6510009765625, + "KL/mean": -425.4344482421875, + "KL/rejected_KL_mean": -511.21783447265625, + "KL/std": 242.4154510498047, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0017284993082284927, + "fcm_dpo/delta": 0.03282516449689865, + "fcm_dpo/margin": 171.56690979003906, + "fcm_dpo/q_t": 0.4304983615875244, + "grad_norm": 33.60206604003906, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.48218047618865967, + "logits/rejected": -0.4797579050064087, + "logps/chosen": -393.5205078125, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -601.9871215820312, + "loss": 1.1616, + "margin_dpo/margin_mean": 171.56692504882812, + "margin_dpo/margin_std": 264.50006103515625, + "step": 361 + }, + { + "KL/chosen_KL_mean": -294.84161376953125, + "KL/mean": -435.40869140625, + "KL/rejected_KL_mean": -575.9757690429688, + "KL/std": 257.6809997558594, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.001707769464701414, + "fcm_dpo/delta": -0.08454330265522003, + "fcm_dpo/margin": 281.1341552734375, + "fcm_dpo/q_t": 0.3883470892906189, + "grad_norm": 28.32308006286621, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.4712877869606018, + "logits/rejected": -0.48378074169158936, + "logps/chosen": -353.4806823730469, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -681.5577392578125, + "loss": 1.0119, + "margin_dpo/margin_mean": 281.1341552734375, + "margin_dpo/margin_std": 271.9013671875, + "step": 362 + }, + { + "KL/chosen_KL_mean": -272.4012451171875, + "KL/mean": -414.70538330078125, + "KL/rejected_KL_mean": -557.009521484375, + "KL/std": 256.60687255859375, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0016879208851605654, + "fcm_dpo/delta": -0.08436906337738037, + "fcm_dpo/margin": 284.60833740234375, + "fcm_dpo/q_t": 0.3883429169654846, + "grad_norm": 24.26263427734375, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.4750707149505615, + "logits/rejected": -0.47332486510276794, + "logps/chosen": -316.9596252441406, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -631.7044677734375, + "loss": 1.0154, + "margin_dpo/margin_mean": 284.60833740234375, + "margin_dpo/margin_std": 282.489013671875, + "step": 363 + }, + { + "KL/chosen_KL_mean": -294.09930419921875, + "KL/mean": -400.2091064453125, + "KL/rejected_KL_mean": -506.31890869140625, + "KL/std": 246.54312133789062, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0016748521011322737, + "fcm_dpo/delta": -0.010238933376967907, + "fcm_dpo/margin": 212.21963500976562, + "fcm_dpo/q_t": 0.41655871272087097, + "grad_norm": 21.17912483215332, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.46631863713264465, + "logits/rejected": -0.4765163064002991, + "logps/chosen": -342.993896484375, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -597.7146606445312, + "loss": 1.1163, + "margin_dpo/margin_mean": 212.21963500976562, + "margin_dpo/margin_std": 283.04998779296875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -283.20660400390625, + "KL/mean": -395.921875, + "KL/rejected_KL_mean": -508.63714599609375, + "KL/std": 256.2212219238281, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0016787885688245296, + "fcm_dpo/delta": 0.022396612912416458, + "fcm_dpo/margin": 225.4305419921875, + "fcm_dpo/q_t": 0.4123465120792389, + "grad_norm": 23.006120681762695, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.448117733001709, + "logits/rejected": -0.4598590135574341, + "logps/chosen": -334.6993408203125, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -601.3388061523438, + "loss": 1.0903, + "margin_dpo/margin_mean": 225.4305419921875, + "margin_dpo/margin_std": 265.99993896484375, + "step": 365 + }, + { + "KL/chosen_KL_mean": -265.95263671875, + "KL/mean": -379.69970703125, + "KL/rejected_KL_mean": -493.44677734375, + "KL/std": 246.43138122558594, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.0016705328598618507, + "fcm_dpo/delta": -0.03018508106470108, + "fcm_dpo/margin": 227.494140625, + "fcm_dpo/q_t": 0.41167664527893066, + "grad_norm": 27.311767578125, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.44603431224823, + "logits/rejected": -0.4612619876861572, + "logps/chosen": -310.6732177734375, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -576.7572021484375, + "loss": 1.1003, + "margin_dpo/margin_mean": 227.494140625, + "margin_dpo/margin_std": 295.17327880859375, + "step": 366 + }, + { + "KL/chosen_KL_mean": -290.567138671875, + "KL/mean": -385.83441162109375, + "KL/rejected_KL_mean": -481.1016845703125, + "KL/std": 225.00637817382812, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0016743103042244911, + "fcm_dpo/delta": 0.022561630234122276, + "fcm_dpo/margin": 190.5345916748047, + "fcm_dpo/q_t": 0.42398330569267273, + "grad_norm": 25.068416595458984, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.48436498641967773, + "logits/rejected": -0.4692809581756592, + "logps/chosen": -348.9725341796875, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -557.85302734375, + "loss": 1.1316, + "margin_dpo/margin_mean": 190.53457641601562, + "margin_dpo/margin_std": 239.75726318359375, + "step": 367 + }, + { + "KL/chosen_KL_mean": -274.366455078125, + "KL/mean": -427.067626953125, + "KL/rejected_KL_mean": -579.768798828125, + "KL/std": 260.6933288574219, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0016510996501892805, + "fcm_dpo/delta": -0.10977530479431152, + "fcm_dpo/margin": 305.40228271484375, + "fcm_dpo/q_t": 0.3797553479671478, + "grad_norm": 41.33750915527344, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.49827373027801514, + "logits/rejected": -0.5085276365280151, + "logps/chosen": -318.8189697265625, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -678.3240356445312, + "loss": 0.9793, + "margin_dpo/margin_mean": 305.40228271484375, + "margin_dpo/margin_std": 233.13153076171875, + "step": 368 + }, + { + "KL/chosen_KL_mean": -380.1885681152344, + "KL/mean": -459.5096435546875, + "KL/rejected_KL_mean": -538.8306884765625, + "KL/std": 271.7216796875, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0016477121971547604, + "fcm_dpo/delta": 0.036811020225286484, + "fcm_dpo/margin": 158.64215087890625, + "fcm_dpo/q_t": 0.43851014971733093, + "grad_norm": 27.48948860168457, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.47715622186660767, + "logits/rejected": -0.4571559727191925, + "logps/chosen": -451.570068359375, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -630.1265258789062, + "loss": 1.2028, + "margin_dpo/margin_mean": 158.64215087890625, + "margin_dpo/margin_std": 305.65875244140625, + "step": 369 + }, + { + "KL/chosen_KL_mean": -405.1291809082031, + "KL/mean": -476.9594421386719, + "KL/rejected_KL_mean": -548.7897338867188, + "KL/std": 282.4532165527344, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0016537775518372655, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 143.6605682373047, + "fcm_dpo/q_t": 0.444976806640625, + "grad_norm": 35.43803405761719, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.5256311893463135, + "logits/rejected": -0.5194545984268188, + "logps/chosen": -476.7366943359375, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -646.049560546875, + "loss": 1.2278, + "margin_dpo/margin_mean": 143.6605682373047, + "margin_dpo/margin_std": 313.33660888671875, + "step": 370 + }, + { + "KL/chosen_KL_mean": -391.35626220703125, + "KL/mean": -504.91375732421875, + "KL/rejected_KL_mean": -618.47119140625, + "KL/std": 293.90032958984375, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0016638652887195349, + "fcm_dpo/delta": 0.02277124673128128, + "fcm_dpo/margin": 227.11500549316406, + "fcm_dpo/q_t": 0.41387006640434265, + "grad_norm": 24.866744995117188, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.5352590680122375, + "logits/rejected": -0.5203221440315247, + "logps/chosen": -460.770751953125, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -717.6434326171875, + "loss": 1.1144, + "margin_dpo/margin_mean": 227.11500549316406, + "margin_dpo/margin_std": 328.7462158203125, + "step": 371 + }, + { + "KL/chosen_KL_mean": -390.8066711425781, + "KL/mean": -523.05810546875, + "KL/rejected_KL_mean": -655.3095092773438, + "KL/std": 331.9268798828125, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.001651276950724423, + "fcm_dpo/delta": -0.03846225142478943, + "fcm_dpo/margin": 264.50286865234375, + "fcm_dpo/q_t": 0.4036857485771179, + "grad_norm": 33.63286209106445, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.5665595531463623, + "logits/rejected": -0.5583839416503906, + "logps/chosen": -452.6246643066406, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -733.8489990234375, + "loss": 1.0833, + "margin_dpo/margin_mean": 264.50286865234375, + "margin_dpo/margin_std": 384.27978515625, + "step": 372 + }, + { + "KL/chosen_KL_mean": -409.9856262207031, + "KL/mean": -536.2109375, + "KL/rejected_KL_mean": -662.436279296875, + "KL/std": 308.58538818359375, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.00165200000628829, + "fcm_dpo/delta": -0.018165547400712967, + "fcm_dpo/margin": 252.4506378173828, + "fcm_dpo/q_t": 0.4031188488006592, + "grad_norm": 26.066843032836914, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.5387763977050781, + "logits/rejected": -0.526741087436676, + "logps/chosen": -474.2027587890625, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -748.3958740234375, + "loss": 1.0755, + "margin_dpo/margin_mean": 252.4506378173828, + "margin_dpo/margin_std": 304.8489685058594, + "step": 373 + }, + { + "KL/chosen_KL_mean": -354.8533935546875, + "KL/mean": -470.448974609375, + "KL/rejected_KL_mean": -586.0445556640625, + "KL/std": 328.1663818359375, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.0016299440758302808, + "fcm_dpo/delta": -0.03872590884566307, + "fcm_dpo/margin": 231.19114685058594, + "fcm_dpo/q_t": 0.4165589511394501, + "grad_norm": 30.272136688232422, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.5748541355133057, + "logits/rejected": -0.5587773323059082, + "logps/chosen": -425.5035705566406, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -679.6846923828125, + "loss": 1.1293, + "margin_dpo/margin_mean": 231.191162109375, + "margin_dpo/margin_std": 373.0335388183594, + "step": 374 + }, + { + "KL/chosen_KL_mean": -359.8167724609375, + "KL/mean": -473.55682373046875, + "KL/rejected_KL_mean": -587.296875, + "KL/std": 264.8832092285156, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.0016179224476218224, + "fcm_dpo/delta": -0.037014126777648926, + "fcm_dpo/margin": 227.48007202148438, + "fcm_dpo/q_t": 0.41423773765563965, + "grad_norm": 30.85564613342285, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.5151851177215576, + "logits/rejected": -0.5146843194961548, + "logps/chosen": -419.89703369140625, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -676.2351684570312, + "loss": 1.1138, + "margin_dpo/margin_mean": 227.48007202148438, + "margin_dpo/margin_std": 311.9302673339844, + "step": 375 + }, + { + "KL/chosen_KL_mean": -348.69085693359375, + "KL/mean": -486.150390625, + "KL/rejected_KL_mean": -623.6099853515625, + "KL/std": 284.18560791015625, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0016054344596341252, + "fcm_dpo/delta": -0.04336491972208023, + "fcm_dpo/margin": 274.91912841796875, + "fcm_dpo/q_t": 0.39769411087036133, + "grad_norm": 28.340049743652344, + "learning_rate": 2.5e-07, + "logits/chosen": -0.4901260733604431, + "logits/rejected": -0.48154765367507935, + "logps/chosen": -411.35113525390625, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -729.1365966796875, + "loss": 1.0581, + "margin_dpo/margin_mean": 274.91912841796875, + "margin_dpo/margin_std": 328.03564453125, + "step": 376 + }, + { + "KL/chosen_KL_mean": -338.87799072265625, + "KL/mean": -469.91455078125, + "KL/rejected_KL_mean": -600.9511108398438, + "KL/std": 296.74615478515625, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.0016020347829908133, + "fcm_dpo/delta": -0.020754069089889526, + "fcm_dpo/margin": 262.0731201171875, + "fcm_dpo/q_t": 0.4025871157646179, + "grad_norm": 31.146583557128906, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.4690827429294586, + "logits/rejected": -0.4671935737133026, + "logps/chosen": -393.3567199707031, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -699.6544189453125, + "loss": 1.0672, + "margin_dpo/margin_mean": 262.0731201171875, + "margin_dpo/margin_std": 307.7542419433594, + "step": 377 + }, + { + "KL/chosen_KL_mean": -319.8349609375, + "KL/mean": -451.2657470703125, + "KL/rejected_KL_mean": -582.696533203125, + "KL/std": 259.9336242675781, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.0015920358709990978, + "fcm_dpo/delta": -0.019304465502500534, + "fcm_dpo/margin": 262.861572265625, + "fcm_dpo/q_t": 0.40114808082580566, + "grad_norm": 32.4703254699707, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.4593803584575653, + "logits/rejected": -0.4763679504394531, + "logps/chosen": -364.85552978515625, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -670.7435302734375, + "loss": 1.058, + "margin_dpo/margin_mean": 262.8615417480469, + "margin_dpo/margin_std": 280.3841552734375, + "step": 378 + }, + { + "KL/chosen_KL_mean": -348.70294189453125, + "KL/mean": -484.337646484375, + "KL/rejected_KL_mean": -619.9722900390625, + "KL/std": 272.16387939453125, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0015784346032887697, + "fcm_dpo/delta": -0.029811790212988853, + "fcm_dpo/margin": 271.2694396972656, + "fcm_dpo/q_t": 0.4009660482406616, + "grad_norm": 25.104984283447266, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.4522586464881897, + "logits/rejected": -0.470679372549057, + "logps/chosen": -401.885009765625, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -734.2724609375, + "loss": 1.0628, + "margin_dpo/margin_mean": 271.2694091796875, + "margin_dpo/margin_std": 317.9272766113281, + "step": 379 + }, + { + "KL/chosen_KL_mean": -339.8782043457031, + "KL/mean": -488.4520568847656, + "KL/rejected_KL_mean": -637.0260009765625, + "KL/std": 308.487060546875, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.0015668224077671766, + "fcm_dpo/delta": -0.06873907893896103, + "fcm_dpo/margin": 297.14776611328125, + "fcm_dpo/q_t": 0.39164721965789795, + "grad_norm": 23.72435188293457, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.44181495904922485, + "logits/rejected": -0.4671395719051361, + "logps/chosen": -391.2312316894531, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -741.2176513671875, + "loss": 1.0377, + "margin_dpo/margin_mean": 297.14776611328125, + "margin_dpo/margin_std": 335.04541015625, + "step": 380 + }, + { + "KL/chosen_KL_mean": -346.30450439453125, + "KL/mean": -450.3425598144531, + "KL/rejected_KL_mean": -554.3806762695312, + "KL/std": 251.66659545898438, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.0015668608248233795, + "fcm_dpo/delta": 0.02163066901266575, + "fcm_dpo/margin": 208.076171875, + "fcm_dpo/q_t": 0.4247140884399414, + "grad_norm": 26.564653396606445, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4704021215438843, + "logits/rejected": -0.4526394009590149, + "logps/chosen": -404.1075439453125, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -633.60009765625, + "loss": 1.1563, + "margin_dpo/margin_mean": 208.07618713378906, + "margin_dpo/margin_std": 346.003173828125, + "step": 381 + }, + { + "KL/chosen_KL_mean": -339.00323486328125, + "KL/mean": -467.90478515625, + "KL/rejected_KL_mean": -596.8063354492188, + "KL/std": 246.88833618164062, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0015656345058232546, + "fcm_dpo/delta": -0.0037797787226736546, + "fcm_dpo/margin": 257.8031005859375, + "fcm_dpo/q_t": 0.40494123101234436, + "grad_norm": 24.68714141845703, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.5269554853439331, + "logits/rejected": -0.5323977470397949, + "logps/chosen": -405.0235595703125, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -707.5164794921875, + "loss": 1.0655, + "margin_dpo/margin_mean": 257.8031005859375, + "margin_dpo/margin_std": 270.8031005859375, + "step": 382 + }, + { + "KL/chosen_KL_mean": -353.4311828613281, + "KL/mean": -476.3453369140625, + "KL/rejected_KL_mean": -599.259521484375, + "KL/std": 269.9713134765625, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0015677779447287321, + "fcm_dpo/delta": 0.015177578665316105, + "fcm_dpo/margin": 245.82827758789062, + "fcm_dpo/q_t": 0.4107585847377777, + "grad_norm": 23.97294807434082, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.4577069878578186, + "logits/rejected": -0.4669598639011383, + "logps/chosen": -403.82269287109375, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -692.975341796875, + "loss": 1.11, + "margin_dpo/margin_mean": 245.82827758789062, + "margin_dpo/margin_std": 354.49835205078125, + "step": 383 + }, + { + "KL/chosen_KL_mean": -356.974609375, + "KL/mean": -455.6815185546875, + "KL/rejected_KL_mean": -554.388427734375, + "KL/std": 250.4655303955078, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0015724662225693464, + "fcm_dpo/delta": 0.012907405383884907, + "fcm_dpo/margin": 197.41383361816406, + "fcm_dpo/q_t": 0.42566782236099243, + "grad_norm": 28.231067657470703, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.529116153717041, + "logits/rejected": -0.5218390226364136, + "logps/chosen": -409.0207214355469, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -640.1493530273438, + "loss": 1.1317, + "margin_dpo/margin_mean": 197.41383361816406, + "margin_dpo/margin_std": 228.51565551757812, + "step": 384 + }, + { + "KL/chosen_KL_mean": -329.0858154296875, + "KL/mean": -449.4140625, + "KL/rejected_KL_mean": -569.7423095703125, + "KL/std": 219.86566162109375, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0015800036489963531, + "fcm_dpo/delta": 0.020530305802822113, + "fcm_dpo/margin": 240.656494140625, + "fcm_dpo/q_t": 0.4094512462615967, + "grad_norm": 33.3217658996582, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.5464999675750732, + "logits/rejected": -0.5235676765441895, + "logps/chosen": -394.63800048828125, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -647.5702514648438, + "loss": 1.0769, + "margin_dpo/margin_mean": 240.656494140625, + "margin_dpo/margin_std": 241.36306762695312, + "step": 385 + }, + { + "KL/chosen_KL_mean": -354.4156188964844, + "KL/mean": -489.7166442871094, + "KL/rejected_KL_mean": -625.0177001953125, + "KL/std": 285.1136169433594, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0015758727677166462, + "fcm_dpo/delta": -0.027603352442383766, + "fcm_dpo/margin": 270.60205078125, + "fcm_dpo/q_t": 0.4016386568546295, + "grad_norm": 22.22235870361328, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.5049442648887634, + "logits/rejected": -0.4999094605445862, + "logps/chosen": -412.6374816894531, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -717.3450927734375, + "loss": 1.0688, + "margin_dpo/margin_mean": 270.60205078125, + "margin_dpo/margin_std": 334.1772766113281, + "step": 386 + }, + { + "KL/chosen_KL_mean": -382.9866943359375, + "KL/mean": -493.877685546875, + "KL/rejected_KL_mean": -604.7686767578125, + "KL/std": 269.3741149902344, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0015690994914621115, + "fcm_dpo/delta": -0.010093586519360542, + "fcm_dpo/margin": 221.78199768066406, + "fcm_dpo/q_t": 0.4175119400024414, + "grad_norm": 30.99074935913086, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.5496755838394165, + "logits/rejected": -0.5332236289978027, + "logps/chosen": -449.4061279296875, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -696.9378662109375, + "loss": 1.1127, + "margin_dpo/margin_mean": 221.78201293945312, + "margin_dpo/margin_std": 269.1995544433594, + "step": 387 + }, + { + "KL/chosen_KL_mean": -366.6164245605469, + "KL/mean": -517.356201171875, + "KL/rejected_KL_mean": -668.0960083007812, + "KL/std": 309.20648193359375, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0015488180797547102, + "fcm_dpo/delta": -0.07052278518676758, + "fcm_dpo/margin": 301.47955322265625, + "fcm_dpo/q_t": 0.39403438568115234, + "grad_norm": 27.048242568969727, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.44531822204589844, + "logits/rejected": -0.4547153115272522, + "logps/chosen": -416.7458801269531, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -772.529052734375, + "loss": 1.0431, + "margin_dpo/margin_mean": 301.47955322265625, + "margin_dpo/margin_std": 359.73187255859375, + "step": 388 + }, + { + "KL/chosen_KL_mean": -402.54791259765625, + "KL/mean": -524.427490234375, + "KL/rejected_KL_mean": -646.3070068359375, + "KL/std": 301.0526428222656, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.0015501437010243535, + "fcm_dpo/delta": 0.022997096180915833, + "fcm_dpo/margin": 243.75909423828125, + "fcm_dpo/q_t": 0.4124375581741333, + "grad_norm": 33.99372482299805, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.5067400932312012, + "logits/rejected": -0.49958211183547974, + "logps/chosen": -460.4544677734375, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -724.2215576171875, + "loss": 1.1056, + "margin_dpo/margin_mean": 243.75909423828125, + "margin_dpo/margin_std": 330.5262756347656, + "step": 389 + }, + { + "KL/chosen_KL_mean": -383.2017822265625, + "KL/mean": -508.2411804199219, + "KL/rejected_KL_mean": -633.2805786132812, + "KL/std": 293.96051025390625, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.0015520667657256126, + "fcm_dpo/delta": 0.012166431173682213, + "fcm_dpo/margin": 250.07879638671875, + "fcm_dpo/q_t": 0.4134998619556427, + "grad_norm": 29.50570297241211, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.4670882225036621, + "logits/rejected": -0.4587477445602417, + "logps/chosen": -432.427734375, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -718.8087768554688, + "loss": 1.1134, + "margin_dpo/margin_mean": 250.0787811279297, + "margin_dpo/margin_std": 376.0643005371094, + "step": 390 + }, + { + "KL/chosen_KL_mean": -395.19830322265625, + "KL/mean": -464.30908203125, + "KL/rejected_KL_mean": -533.4198608398438, + "KL/std": 285.359130859375, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0015580422477796674, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 138.22158813476562, + "fcm_dpo/q_t": 0.4491511881351471, + "grad_norm": 48.606056213378906, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.4878583550453186, + "logits/rejected": -0.4785499572753906, + "logps/chosen": -459.5279541015625, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -620.1580810546875, + "loss": 1.2369, + "margin_dpo/margin_mean": 138.22158813476562, + "margin_dpo/margin_std": 301.6778869628906, + "step": 391 + }, + { + "KL/chosen_KL_mean": -327.66204833984375, + "KL/mean": -471.5645751953125, + "KL/rejected_KL_mean": -615.4671020507812, + "KL/std": 287.97943115234375, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0015518094878643751, + "fcm_dpo/delta": -0.04885515943169594, + "fcm_dpo/margin": 287.8050537109375, + "fcm_dpo/q_t": 0.39560776948928833, + "grad_norm": 26.586740493774414, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.4810331165790558, + "logits/rejected": -0.4747980237007141, + "logps/chosen": -381.1660461425781, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -717.8129272460938, + "loss": 1.0352, + "margin_dpo/margin_mean": 287.8050537109375, + "margin_dpo/margin_std": 284.23077392578125, + "step": 392 + }, + { + "KL/chosen_KL_mean": -320.0975646972656, + "KL/mean": -430.493896484375, + "KL/rejected_KL_mean": -540.8902587890625, + "KL/std": 285.0341491699219, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0015467579942196608, + "fcm_dpo/delta": 0.012510502710938454, + "fcm_dpo/margin": 220.79266357421875, + "fcm_dpo/q_t": 0.42353495955467224, + "grad_norm": 26.941801071166992, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.4879850149154663, + "logits/rejected": -0.48997747898101807, + "logps/chosen": -366.57147216796875, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -612.859130859375, + "loss": 1.1434, + "margin_dpo/margin_mean": 220.79266357421875, + "margin_dpo/margin_std": 361.098876953125, + "step": 393 + }, + { + "KL/chosen_KL_mean": -359.8177795410156, + "KL/mean": -479.6298828125, + "KL/rejected_KL_mean": -599.4420166015625, + "KL/std": 297.6522521972656, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.001551133580505848, + "fcm_dpo/delta": 0.02939993515610695, + "fcm_dpo/margin": 239.62420654296875, + "fcm_dpo/q_t": 0.41440391540527344, + "grad_norm": 25.9660587310791, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.440055787563324, + "logits/rejected": -0.44040584564208984, + "logps/chosen": -412.7293395996094, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -690.2646484375, + "loss": 1.1046, + "margin_dpo/margin_mean": 239.6241912841797, + "margin_dpo/margin_std": 314.5859375, + "step": 394 + }, + { + "KL/chosen_KL_mean": -357.7408447265625, + "KL/mean": -487.4169616699219, + "KL/rejected_KL_mean": -617.0930786132812, + "KL/std": 293.8997802734375, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.0015505390474572778, + "fcm_dpo/delta": -0.0024265367537736893, + "fcm_dpo/margin": 259.3522033691406, + "fcm_dpo/q_t": 0.40790221095085144, + "grad_norm": 30.717853546142578, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.4773879051208496, + "logits/rejected": -0.47425657510757446, + "logps/chosen": -420.2869873046875, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -700.8756713867188, + "loss": 1.0901, + "margin_dpo/margin_mean": 259.3522033691406, + "margin_dpo/margin_std": 342.5379638671875, + "step": 395 + }, + { + "KL/chosen_KL_mean": -358.50946044921875, + "KL/mean": -471.33978271484375, + "KL/rejected_KL_mean": -584.1700439453125, + "KL/std": 282.5489196777344, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.001564518315717578, + "fcm_dpo/delta": 0.0486970953643322, + "fcm_dpo/margin": 225.66064453125, + "fcm_dpo/q_t": 0.4169620871543884, + "grad_norm": 23.7697811126709, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.47971564531326294, + "logits/rejected": -0.4619836211204529, + "logps/chosen": -427.5054016113281, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -672.8167724609375, + "loss": 1.1031, + "margin_dpo/margin_mean": 225.66064453125, + "margin_dpo/margin_std": 260.45703125, + "step": 396 + }, + { + "KL/chosen_KL_mean": -338.2709655761719, + "KL/mean": -485.96466064453125, + "KL/rejected_KL_mean": -633.658447265625, + "KL/std": 280.13671875, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0015507256612181664, + "fcm_dpo/delta": -0.06130717694759369, + "fcm_dpo/margin": 295.387451171875, + "fcm_dpo/q_t": 0.39238476753234863, + "grad_norm": 33.7121467590332, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.46764034032821655, + "logits/rejected": -0.4777331054210663, + "logps/chosen": -399.5481262207031, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -736.7745361328125, + "loss": 1.0284, + "margin_dpo/margin_mean": 295.3874816894531, + "margin_dpo/margin_std": 291.95166015625, + "step": 397 + }, + { + "KL/chosen_KL_mean": -368.6435852050781, + "KL/mean": -502.7423095703125, + "KL/rejected_KL_mean": -636.841064453125, + "KL/std": 298.6142272949219, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0015470728976652026, + "fcm_dpo/delta": -0.015574896708130836, + "fcm_dpo/margin": 268.1974792480469, + "fcm_dpo/q_t": 0.40525174140930176, + "grad_norm": 20.82304573059082, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.5198394060134888, + "logits/rejected": -0.5158972144126892, + "logps/chosen": -436.7951354980469, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -745.3646850585938, + "loss": 1.0804, + "margin_dpo/margin_mean": 268.1974792480469, + "margin_dpo/margin_std": 349.257568359375, + "step": 398 + }, + { + "KL/chosen_KL_mean": -314.2664794921875, + "KL/mean": -430.8349609375, + "KL/rejected_KL_mean": -547.4034423828125, + "KL/std": 256.01409912109375, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0015507233329117298, + "fcm_dpo/delta": 0.03983534872531891, + "fcm_dpo/margin": 233.13697814941406, + "fcm_dpo/q_t": 0.41602247953414917, + "grad_norm": 25.84345245361328, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.5000230073928833, + "logits/rejected": -0.46655428409576416, + "logps/chosen": -375.1562805175781, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -625.3690185546875, + "loss": 1.1171, + "margin_dpo/margin_mean": 233.13697814941406, + "margin_dpo/margin_std": 322.0264587402344, + "step": 399 + }, + { + "KL/chosen_KL_mean": -303.71539306640625, + "KL/mean": -474.6365966796875, + "KL/rejected_KL_mean": -645.5578002929688, + "KL/std": 277.0064697265625, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.001526598702184856, + "fcm_dpo/delta": -0.12884403765201569, + "fcm_dpo/margin": 341.8423767089844, + "fcm_dpo/q_t": 0.3768342435359955, + "grad_norm": 32.650108337402344, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.5031575560569763, + "logits/rejected": -0.5052364468574524, + "logps/chosen": -367.3590087890625, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -750.8104858398438, + "loss": 0.9823, + "margin_dpo/margin_mean": 341.8423767089844, + "margin_dpo/margin_std": 307.00347900390625, + "step": 400 + }, + { + "KL/chosen_KL_mean": -357.942626953125, + "KL/mean": -453.41461181640625, + "KL/rejected_KL_mean": -548.8865966796875, + "KL/std": 289.3387145996094, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0015271191950887442, + "fcm_dpo/delta": 0.026330213993787766, + "fcm_dpo/margin": 190.9439239501953, + "fcm_dpo/q_t": 0.4342236816883087, + "grad_norm": 27.997446060180664, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.4633658528327942, + "logits/rejected": -0.45505958795547485, + "logps/chosen": -415.1056823730469, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -632.6790771484375, + "loss": 1.1989, + "margin_dpo/margin_mean": 190.94393920898438, + "margin_dpo/margin_std": 386.1468811035156, + "step": 401 + }, + { + "KL/chosen_KL_mean": -260.52886962890625, + "KL/mean": -441.70770263671875, + "KL/rejected_KL_mean": -622.886474609375, + "KL/std": 312.40179443359375, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.0014937083469703794, + "fcm_dpo/delta": -0.1493334174156189, + "fcm_dpo/margin": 362.35760498046875, + "fcm_dpo/q_t": 0.37349700927734375, + "grad_norm": 30.656410217285156, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.5205470323562622, + "logits/rejected": -0.5141332149505615, + "logps/chosen": -311.26922607421875, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -703.9324951171875, + "loss": 0.9663, + "margin_dpo/margin_mean": 362.35760498046875, + "margin_dpo/margin_std": 315.6654968261719, + "step": 402 + }, + { + "KL/chosen_KL_mean": -286.92822265625, + "KL/mean": -419.6138916015625, + "KL/rejected_KL_mean": -552.299560546875, + "KL/std": 289.564453125, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0014832005836069584, + "fcm_dpo/delta": 0.00666351243853569, + "fcm_dpo/margin": 265.3713073730469, + "fcm_dpo/q_t": 0.4080093801021576, + "grad_norm": 31.32441520690918, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.5001975297927856, + "logits/rejected": -0.503011167049408, + "logps/chosen": -334.0755615234375, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -629.566162109375, + "loss": 1.0788, + "margin_dpo/margin_mean": 265.37127685546875, + "margin_dpo/margin_std": 300.79052734375, + "step": 403 + }, + { + "KL/chosen_KL_mean": -319.355712890625, + "KL/mean": -451.35357666015625, + "KL/rejected_KL_mean": -583.3514404296875, + "KL/std": 275.60345458984375, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0014838875504210591, + "fcm_dpo/delta": 0.0085657499730587, + "fcm_dpo/margin": 263.9956970214844, + "fcm_dpo/q_t": 0.4091029465198517, + "grad_norm": 34.7021369934082, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.48698821663856506, + "logits/rejected": -0.4795249104499817, + "logps/chosen": -367.2309875488281, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -660.5064086914062, + "loss": 1.096, + "margin_dpo/margin_mean": 263.99566650390625, + "margin_dpo/margin_std": 348.069091796875, + "step": 404 + }, + { + "KL/chosen_KL_mean": -364.86529541015625, + "KL/mean": -471.24188232421875, + "KL/rejected_KL_mean": -577.618408203125, + "KL/std": 304.8662109375, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.0014832655433565378, + "fcm_dpo/delta": -0.02318808250129223, + "fcm_dpo/margin": 212.75315856933594, + "fcm_dpo/q_t": 0.4282769560813904, + "grad_norm": 30.10840606689453, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.48986512422561646, + "logits/rejected": -0.48185962438583374, + "logps/chosen": -430.0281982421875, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -664.8052368164062, + "loss": 1.1751, + "margin_dpo/margin_mean": 212.75315856933594, + "margin_dpo/margin_std": 382.7352294921875, + "step": 405 + }, + { + "KL/chosen_KL_mean": -311.94671630859375, + "KL/mean": -452.7980651855469, + "KL/rejected_KL_mean": -593.6494140625, + "KL/std": 296.5862731933594, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0014813740272074938, + "fcm_dpo/delta": -0.01828046888113022, + "fcm_dpo/margin": 281.70269775390625, + "fcm_dpo/q_t": 0.40377163887023926, + "grad_norm": 30.288841247558594, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.5453722476959229, + "logits/rejected": -0.5596540570259094, + "logps/chosen": -361.6875305175781, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -685.72802734375, + "loss": 1.071, + "margin_dpo/margin_mean": 281.7026672363281, + "margin_dpo/margin_std": 337.3284606933594, + "step": 406 + }, + { + "KL/chosen_KL_mean": -338.5423583984375, + "KL/mean": -430.4468994140625, + "KL/rejected_KL_mean": -522.3515014648438, + "KL/std": 243.62417602539062, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0014756100717931986, + "fcm_dpo/delta": 0.008026116527616978, + "fcm_dpo/margin": 183.8091278076172, + "fcm_dpo/q_t": 0.43563488125801086, + "grad_norm": 29.608999252319336, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.5653454065322876, + "logits/rejected": -0.5378514528274536, + "logps/chosen": -394.873046875, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -599.8635864257812, + "loss": 1.1903, + "margin_dpo/margin_mean": 183.80914306640625, + "margin_dpo/margin_std": 320.85845947265625, + "step": 407 + }, + { + "KL/chosen_KL_mean": -344.60906982421875, + "KL/mean": -443.21844482421875, + "KL/rejected_KL_mean": -541.8278198242188, + "KL/std": 234.69036865234375, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.001483906526118517, + "fcm_dpo/delta": 0.04792923107743263, + "fcm_dpo/margin": 197.21871948242188, + "fcm_dpo/q_t": 0.4303654432296753, + "grad_norm": 26.166357040405273, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.5701711773872375, + "logits/rejected": -0.5480648279190063, + "logps/chosen": -414.3984069824219, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -631.9247436523438, + "loss": 1.1504, + "margin_dpo/margin_mean": 197.21871948242188, + "margin_dpo/margin_std": 257.93182373046875, + "step": 408 + }, + { + "KL/chosen_KL_mean": -327.5982971191406, + "KL/mean": -426.26934814453125, + "KL/rejected_KL_mean": -524.9404296875, + "KL/std": 246.71730041503906, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0014951603952795267, + "fcm_dpo/delta": 0.02769964560866356, + "fcm_dpo/margin": 197.34205627441406, + "fcm_dpo/q_t": 0.43003880977630615, + "grad_norm": 31.87028694152832, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.5277822017669678, + "logits/rejected": -0.5139531493186951, + "logps/chosen": -394.9157409667969, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -609.8447265625, + "loss": 1.1537, + "margin_dpo/margin_mean": 197.34207153320312, + "margin_dpo/margin_std": 273.16021728515625, + "step": 409 + }, + { + "KL/chosen_KL_mean": -312.26251220703125, + "KL/mean": -432.8775329589844, + "KL/rejected_KL_mean": -553.4925537109375, + "KL/std": 257.06488037109375, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0015057341661304235, + "fcm_dpo/delta": 0.0381709560751915, + "fcm_dpo/margin": 241.22999572753906, + "fcm_dpo/q_t": 0.4139382541179657, + "grad_norm": 31.423391342163086, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.583343505859375, + "logits/rejected": -0.5878040194511414, + "logps/chosen": -363.7279052734375, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -636.6915283203125, + "loss": 1.1005, + "margin_dpo/margin_mean": 241.23001098632812, + "margin_dpo/margin_std": 286.34918212890625, + "step": 410 + }, + { + "KL/chosen_KL_mean": -334.853515625, + "KL/mean": -451.3477478027344, + "KL/rejected_KL_mean": -567.8419799804688, + "KL/std": 277.9627685546875, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0014994360972195864, + "fcm_dpo/delta": -0.037723273038864136, + "fcm_dpo/margin": 232.98843383789062, + "fcm_dpo/q_t": 0.4192880690097809, + "grad_norm": 29.51055335998535, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.5538594722747803, + "logits/rejected": -0.5675029754638672, + "logps/chosen": -387.1607971191406, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -648.5369262695312, + "loss": 1.1232, + "margin_dpo/margin_mean": 232.98843383789062, + "margin_dpo/margin_std": 317.76287841796875, + "step": 411 + }, + { + "KL/chosen_KL_mean": -351.8658447265625, + "KL/mean": -472.458251953125, + "KL/rejected_KL_mean": -593.0506591796875, + "KL/std": 276.5720520019531, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.001508728601038456, + "fcm_dpo/delta": 0.03745199739933014, + "fcm_dpo/margin": 241.184814453125, + "fcm_dpo/q_t": 0.41510653495788574, + "grad_norm": 40.6356086730957, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.6174789071083069, + "logits/rejected": -0.651949942111969, + "logps/chosen": -405.00994873046875, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -693.1114501953125, + "loss": 1.1077, + "margin_dpo/margin_mean": 241.184814453125, + "margin_dpo/margin_std": 311.7869873046875, + "step": 412 + }, + { + "KL/chosen_KL_mean": -396.29156494140625, + "KL/mean": -521.27392578125, + "KL/rejected_KL_mean": -646.2562255859375, + "KL/std": 295.3948974609375, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0015183680225163698, + "fcm_dpo/delta": 0.021154742687940598, + "fcm_dpo/margin": 249.96466064453125, + "fcm_dpo/q_t": 0.4103488624095917, + "grad_norm": 33.99179458618164, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.5617023706436157, + "logits/rejected": -0.5732187628746033, + "logps/chosen": -457.87353515625, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -745.7296142578125, + "loss": 1.0991, + "margin_dpo/margin_mean": 249.96466064453125, + "margin_dpo/margin_std": 322.1083068847656, + "step": 413 + }, + { + "KL/chosen_KL_mean": -384.6965026855469, + "KL/mean": -520.0925903320312, + "KL/rejected_KL_mean": -655.4886474609375, + "KL/std": 290.1815185546875, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.001506768399849534, + "fcm_dpo/delta": -0.06866828352212906, + "fcm_dpo/margin": 270.7922058105469, + "fcm_dpo/q_t": 0.4045364558696747, + "grad_norm": 32.15410232543945, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.5434101819992065, + "logits/rejected": -0.5386463403701782, + "logps/chosen": -431.3280029296875, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -743.1351928710938, + "loss": 1.0832, + "margin_dpo/margin_mean": 270.7922058105469, + "margin_dpo/margin_std": 336.496826171875, + "step": 414 + }, + { + "KL/chosen_KL_mean": -420.4744873046875, + "KL/mean": -524.1064453125, + "KL/rejected_KL_mean": -627.7384033203125, + "KL/std": 292.1966552734375, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0014992853393778205, + "fcm_dpo/delta": 0.009558776393532753, + "fcm_dpo/margin": 207.26390075683594, + "fcm_dpo/q_t": 0.4268529713153839, + "grad_norm": 26.902969360351562, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.5989441871643066, + "logits/rejected": -0.5950357913970947, + "logps/chosen": -499.0927734375, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -728.2159423828125, + "loss": 1.1543, + "margin_dpo/margin_mean": 207.26388549804688, + "margin_dpo/margin_std": 317.74627685546875, + "step": 415 + }, + { + "KL/chosen_KL_mean": -389.749755859375, + "KL/mean": -551.880126953125, + "KL/rejected_KL_mean": -714.010498046875, + "KL/std": 322.28509521484375, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.0014837831258773804, + "fcm_dpo/delta": -0.08514563739299774, + "fcm_dpo/margin": 324.26068115234375, + "fcm_dpo/q_t": 0.3874993622303009, + "grad_norm": 35.14958572387695, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.6202210187911987, + "logits/rejected": -0.6337435841560364, + "logps/chosen": -448.0289001464844, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -804.5792236328125, + "loss": 1.0148, + "margin_dpo/margin_mean": 324.26068115234375, + "margin_dpo/margin_std": 318.9561767578125, + "step": 416 + }, + { + "KL/chosen_KL_mean": -401.2996826171875, + "KL/mean": -544.381103515625, + "KL/rejected_KL_mean": -687.4625854492188, + "KL/std": 312.311767578125, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.001471617491915822, + "fcm_dpo/delta": -0.02207181230187416, + "fcm_dpo/margin": 286.16290283203125, + "fcm_dpo/q_t": 0.4012377858161926, + "grad_norm": 32.51115036010742, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.6354202628135681, + "logits/rejected": -0.6251734495162964, + "logps/chosen": -451.4984130859375, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -755.6143798828125, + "loss": 1.0587, + "margin_dpo/margin_mean": 286.16290283203125, + "margin_dpo/margin_std": 311.3736267089844, + "step": 417 + }, + { + "KL/chosen_KL_mean": -441.1939392089844, + "KL/mean": -581.7313232421875, + "KL/rejected_KL_mean": -722.2687377929688, + "KL/std": 351.055908203125, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.0014539003605023026, + "fcm_dpo/delta": -0.09338800609111786, + "fcm_dpo/margin": 281.07476806640625, + "fcm_dpo/q_t": 0.40845057368278503, + "grad_norm": 36.12443923950195, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.6652587652206421, + "logits/rejected": -0.6492989659309387, + "logps/chosen": -523.1724243164062, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -826.960205078125, + "loss": 1.1087, + "margin_dpo/margin_mean": 281.07476806640625, + "margin_dpo/margin_std": 412.9871520996094, + "step": 418 + }, + { + "KL/chosen_KL_mean": -400.76947021484375, + "KL/mean": -561.7467651367188, + "KL/rejected_KL_mean": -722.72412109375, + "KL/std": 312.488037109375, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0014278480084612966, + "fcm_dpo/delta": -0.06254878640174866, + "fcm_dpo/margin": 321.9546203613281, + "fcm_dpo/q_t": 0.3929612338542938, + "grad_norm": 27.233367919921875, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.5777145028114319, + "logits/rejected": -0.5806939601898193, + "logps/chosen": -453.7181091308594, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -814.3071899414062, + "loss": 1.0435, + "margin_dpo/margin_mean": 321.9546203613281, + "margin_dpo/margin_std": 368.0567626953125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -505.229736328125, + "KL/mean": -595.7236328125, + "KL/rejected_KL_mean": -686.217529296875, + "KL/std": 324.6322326660156, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0014224171172827482, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.98785400390625, + "fcm_dpo/q_t": 0.4414908289909363, + "grad_norm": 63.34467315673828, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.656416654586792, + "logits/rejected": -0.622793436050415, + "logps/chosen": -582.9996337890625, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -755.5374145507812, + "loss": 1.2365, + "margin_dpo/margin_mean": 180.9878692626953, + "margin_dpo/margin_std": 436.6684265136719, + "step": 420 + }, + { + "KL/chosen_KL_mean": -409.0378112792969, + "KL/mean": -520.5367431640625, + "KL/rejected_KL_mean": -632.0357055664062, + "KL/std": 321.05474853515625, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.001422304194420576, + "fcm_dpo/delta": -0.0007933757733553648, + "fcm_dpo/margin": 222.99789428710938, + "fcm_dpo/q_t": 0.42396125197410583, + "grad_norm": 30.770769119262695, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.6233694553375244, + "logits/rejected": -0.6090277433395386, + "logps/chosen": -462.8036804199219, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -721.317138671875, + "loss": 1.1475, + "margin_dpo/margin_mean": 222.99790954589844, + "margin_dpo/margin_std": 320.5152587890625, + "step": 421 + }, + { + "KL/chosen_KL_mean": -459.16217041015625, + "KL/mean": -582.953857421875, + "KL/rejected_KL_mean": -706.74560546875, + "KL/std": 318.14837646484375, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0014313666615635157, + "fcm_dpo/delta": 0.047315068542957306, + "fcm_dpo/margin": 247.58340454101562, + "fcm_dpo/q_t": 0.41730841994285583, + "grad_norm": 33.53703308105469, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.6303104162216187, + "logits/rejected": -0.633635938167572, + "logps/chosen": -527.7959594726562, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -794.609130859375, + "loss": 1.1296, + "margin_dpo/margin_mean": 247.58340454101562, + "margin_dpo/margin_std": 371.22186279296875, + "step": 422 + }, + { + "KL/chosen_KL_mean": -412.6064758300781, + "KL/mean": -555.1243896484375, + "KL/rejected_KL_mean": -697.6422119140625, + "KL/std": 303.797119140625, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0014340131310746074, + "fcm_dpo/delta": -0.00911680981516838, + "fcm_dpo/margin": 285.03582763671875, + "fcm_dpo/q_t": 0.4042406678199768, + "grad_norm": 35.22314453125, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.6187890768051147, + "logits/rejected": -0.6278023719787598, + "logps/chosen": -467.5967712402344, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -783.9488525390625, + "loss": 1.0685, + "margin_dpo/margin_mean": 285.03582763671875, + "margin_dpo/margin_std": 320.91790771484375, + "step": 423 + }, + { + "KL/chosen_KL_mean": -375.51739501953125, + "KL/mean": -501.42657470703125, + "KL/rejected_KL_mean": -627.3358154296875, + "KL/std": 289.5096740722656, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0014271767577156425, + "fcm_dpo/delta": -0.020702006295323372, + "fcm_dpo/margin": 251.81838989257812, + "fcm_dpo/q_t": 0.4167898893356323, + "grad_norm": 43.38511657714844, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.5883015990257263, + "logits/rejected": -0.5653923749923706, + "logps/chosen": -431.5293273925781, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -693.8147583007812, + "loss": 1.1178, + "margin_dpo/margin_mean": 251.81838989257812, + "margin_dpo/margin_std": 348.22515869140625, + "step": 424 + }, + { + "KL/chosen_KL_mean": -416.1014709472656, + "KL/mean": -519.8480834960938, + "KL/rejected_KL_mean": -623.5947265625, + "KL/std": 283.3201904296875, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.0014319305773824453, + "fcm_dpo/delta": 0.03319881483912468, + "fcm_dpo/margin": 207.49319458007812, + "fcm_dpo/q_t": 0.42925697565078735, + "grad_norm": 38.28807830810547, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.6049838066101074, + "logits/rejected": -0.6066217422485352, + "logps/chosen": -462.970458984375, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -719.5201416015625, + "loss": 1.1708, + "margin_dpo/margin_mean": 207.49319458007812, + "margin_dpo/margin_std": 347.69024658203125, + "step": 425 + }, + { + "KL/chosen_KL_mean": -383.1138916015625, + "KL/mean": -508.259033203125, + "KL/rejected_KL_mean": -633.4041137695312, + "KL/std": 279.45343017578125, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0014334451407194138, + "fcm_dpo/delta": -0.02259773574769497, + "fcm_dpo/margin": 250.29025268554688, + "fcm_dpo/q_t": 0.41526973247528076, + "grad_norm": 24.808752059936523, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.57851243019104, + "logits/rejected": -0.5479116439819336, + "logps/chosen": -459.69744873046875, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -714.6707153320312, + "loss": 1.108, + "margin_dpo/margin_mean": 250.2902374267578, + "margin_dpo/margin_std": 307.17913818359375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -363.2721862792969, + "KL/mean": -454.5751953125, + "KL/rejected_KL_mean": -545.878173828125, + "KL/std": 248.4161834716797, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0014302058843895793, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.60604858398438, + "fcm_dpo/q_t": 0.438289612531662, + "grad_norm": 25.381738662719727, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.6155461072921753, + "logits/rejected": -0.6013084650039673, + "logps/chosen": -428.12603759765625, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -624.4442138671875, + "loss": 1.187, + "margin_dpo/margin_mean": 182.60604858398438, + "margin_dpo/margin_std": 300.3623352050781, + "step": 427 + }, + { + "KL/chosen_KL_mean": -443.7192687988281, + "KL/mean": -574.9083862304688, + "KL/rejected_KL_mean": -706.0974731445312, + "KL/std": 323.1051940917969, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.001430986332707107, + "fcm_dpo/delta": 0.02536536194384098, + "fcm_dpo/margin": 262.3781433105469, + "fcm_dpo/q_t": 0.4134228527545929, + "grad_norm": 29.857704162597656, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.6007826328277588, + "logits/rejected": -0.6077265739440918, + "logps/chosen": -506.35595703125, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -809.3792724609375, + "loss": 1.1212, + "margin_dpo/margin_mean": 262.378173828125, + "margin_dpo/margin_std": 393.4762878417969, + "step": 428 + }, + { + "KL/chosen_KL_mean": -455.1191711425781, + "KL/mean": -551.5887451171875, + "KL/rejected_KL_mean": -648.058349609375, + "KL/std": 302.576416015625, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.001437472295947373, + "fcm_dpo/delta": -5.257836892269552e-05, + "fcm_dpo/margin": 192.93911743164062, + "fcm_dpo/q_t": 0.4354352653026581, + "grad_norm": 47.41379928588867, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.6075571179389954, + "logits/rejected": -0.5939396619796753, + "logps/chosen": -536.3531494140625, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -739.853271484375, + "loss": 1.1985, + "margin_dpo/margin_mean": 192.93911743164062, + "margin_dpo/margin_std": 364.7570495605469, + "step": 429 + }, + { + "KL/chosen_KL_mean": -362.6143493652344, + "KL/mean": -470.66693115234375, + "KL/rejected_KL_mean": -578.719482421875, + "KL/std": 265.7591247558594, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0014425687259063125, + "fcm_dpo/delta": 0.035381413996219635, + "fcm_dpo/margin": 216.10516357421875, + "fcm_dpo/q_t": 0.4267041087150574, + "grad_norm": 31.38045883178711, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.6202624440193176, + "logits/rejected": -0.6181871891021729, + "logps/chosen": -423.53466796875, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -683.142333984375, + "loss": 1.1414, + "margin_dpo/margin_mean": 216.10516357421875, + "margin_dpo/margin_std": 294.30303955078125, + "step": 430 + }, + { + "KL/chosen_KL_mean": -335.54156494140625, + "KL/mean": -486.19378662109375, + "KL/rejected_KL_mean": -636.845947265625, + "KL/std": 289.29962158203125, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.0014384451787918806, + "fcm_dpo/delta": -0.034993626177310944, + "fcm_dpo/margin": 301.3044128417969, + "fcm_dpo/q_t": 0.39822471141815186, + "grad_norm": 35.380401611328125, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.5965731739997864, + "logits/rejected": -0.6042895913124084, + "logps/chosen": -392.89031982421875, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -729.6861572265625, + "loss": 1.0504, + "margin_dpo/margin_mean": 301.3044128417969, + "margin_dpo/margin_std": 316.9186706542969, + "step": 431 + }, + { + "KL/chosen_KL_mean": -333.36083984375, + "KL/mean": -480.9893798828125, + "KL/rejected_KL_mean": -628.617919921875, + "KL/std": 281.6934814453125, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.0014400999061763287, + "fcm_dpo/delta": -0.026835039258003235, + "fcm_dpo/margin": 295.25701904296875, + "fcm_dpo/q_t": 0.40007904171943665, + "grad_norm": 43.686588287353516, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.5722031593322754, + "logits/rejected": -0.5684648156166077, + "logps/chosen": -404.4356689453125, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -727.1974487304688, + "loss": 1.0482, + "margin_dpo/margin_mean": 295.25701904296875, + "margin_dpo/margin_std": 282.6282043457031, + "step": 432 + }, + { + "KL/chosen_KL_mean": -438.1981201171875, + "KL/mean": -534.46923828125, + "KL/rejected_KL_mean": -630.7403564453125, + "KL/std": 284.1728820800781, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0014369389973580837, + "fcm_dpo/delta": 0.04911511018872261, + "fcm_dpo/margin": 192.54226684570312, + "fcm_dpo/q_t": 0.4334104657173157, + "grad_norm": 31.801271438598633, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.6090582609176636, + "logits/rejected": -0.5987488031387329, + "logps/chosen": -496.4713134765625, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -726.6912841796875, + "loss": 1.1848, + "margin_dpo/margin_mean": 192.54226684570312, + "margin_dpo/margin_std": 334.10333251953125, + "step": 433 + }, + { + "KL/chosen_KL_mean": -376.7767333984375, + "KL/mean": -499.2869567871094, + "KL/rejected_KL_mean": -621.7972412109375, + "KL/std": 309.23486328125, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0014406443806365132, + "fcm_dpo/delta": -0.011620326898992062, + "fcm_dpo/margin": 245.02041625976562, + "fcm_dpo/q_t": 0.4223513603210449, + "grad_norm": 24.078811645507812, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.6271170377731323, + "logits/rejected": -0.6378086805343628, + "logps/chosen": -438.75042724609375, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -700.2957763671875, + "loss": 1.1329, + "margin_dpo/margin_mean": 245.02041625976562, + "margin_dpo/margin_std": 389.4531555175781, + "step": 434 + }, + { + "KL/chosen_KL_mean": -349.1696472167969, + "KL/mean": -483.0262756347656, + "KL/rejected_KL_mean": -616.8828735351562, + "KL/std": 287.125, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0014420354273170233, + "fcm_dpo/delta": 0.01449626311659813, + "fcm_dpo/margin": 267.7132263183594, + "fcm_dpo/q_t": 0.40962618589401245, + "grad_norm": 35.64519500732422, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.6541143655776978, + "logits/rejected": -0.6440638303756714, + "logps/chosen": -400.67169189453125, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -704.4497680664062, + "loss": 1.0897, + "margin_dpo/margin_mean": 267.71319580078125, + "margin_dpo/margin_std": 318.1378479003906, + "step": 435 + }, + { + "KL/chosen_KL_mean": -376.8411865234375, + "KL/mean": -490.2194519042969, + "KL/rejected_KL_mean": -603.5977172851562, + "KL/std": 257.49566650390625, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0014481113757938147, + "fcm_dpo/delta": 0.011351975612342358, + "fcm_dpo/margin": 226.75653076171875, + "fcm_dpo/q_t": 0.42264947295188904, + "grad_norm": 42.95933151245117, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.6089829206466675, + "logits/rejected": -0.5817937850952148, + "logps/chosen": -448.2449035644531, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -686.325439453125, + "loss": 1.1299, + "margin_dpo/margin_mean": 226.75653076171875, + "margin_dpo/margin_std": 300.5293273925781, + "step": 436 + }, + { + "KL/chosen_KL_mean": -384.22723388671875, + "KL/mean": -497.7215576171875, + "KL/rejected_KL_mean": -611.2158203125, + "KL/std": 255.87161254882812, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0014619443099945784, + "fcm_dpo/delta": 0.07054543495178223, + "fcm_dpo/margin": 226.98867797851562, + "fcm_dpo/q_t": 0.4219895899295807, + "grad_norm": 21.776018142700195, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.649273693561554, + "logits/rejected": -0.633920431137085, + "logps/chosen": -448.97149658203125, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -693.2593994140625, + "loss": 1.1291, + "margin_dpo/margin_mean": 226.98867797851562, + "margin_dpo/margin_std": 303.3597412109375, + "step": 437 + }, + { + "KL/chosen_KL_mean": -408.1085510253906, + "KL/mean": -535.7598266601562, + "KL/rejected_KL_mean": -663.4110107421875, + "KL/std": 293.42010498046875, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0014688025694340467, + "fcm_dpo/delta": 0.025791799649596214, + "fcm_dpo/margin": 255.30247497558594, + "fcm_dpo/q_t": 0.41199779510498047, + "grad_norm": 33.617347717285156, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.6757897138595581, + "logits/rejected": -0.6623625755310059, + "logps/chosen": -467.1272277832031, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -746.4878540039062, + "loss": 1.0987, + "margin_dpo/margin_mean": 255.302490234375, + "margin_dpo/margin_std": 315.57928466796875, + "step": 438 + }, + { + "KL/chosen_KL_mean": -411.8974609375, + "KL/mean": -526.5938720703125, + "KL/rejected_KL_mean": -641.2902221679688, + "KL/std": 303.25787353515625, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.0014591185608878732, + "fcm_dpo/delta": -0.05847563594579697, + "fcm_dpo/margin": 229.39280700683594, + "fcm_dpo/q_t": 0.42281574010849, + "grad_norm": 31.35567855834961, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.6426794528961182, + "logits/rejected": -0.6450637578964233, + "logps/chosen": -465.6815185546875, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -725.2756958007812, + "loss": 1.1458, + "margin_dpo/margin_mean": 229.392822265625, + "margin_dpo/margin_std": 345.57623291015625, + "step": 439 + }, + { + "KL/chosen_KL_mean": -443.8215026855469, + "KL/mean": -578.6383056640625, + "KL/rejected_KL_mean": -713.4550170898438, + "KL/std": 366.0792236328125, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0014637765707448125, + "fcm_dpo/delta": 0.005406741052865982, + "fcm_dpo/margin": 269.633544921875, + "fcm_dpo/q_t": 0.4120253920555115, + "grad_norm": 36.05754089355469, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.726885199546814, + "logits/rejected": -0.7298160791397095, + "logps/chosen": -522.3882446289062, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -809.9527587890625, + "loss": 1.0986, + "margin_dpo/margin_mean": 269.633544921875, + "margin_dpo/margin_std": 378.5799560546875, + "step": 440 + }, + { + "KL/chosen_KL_mean": -515.0677490234375, + "KL/mean": -638.0194091796875, + "KL/rejected_KL_mean": -760.9710693359375, + "KL/std": 372.77166748046875, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.001458245562389493, + "fcm_dpo/delta": -0.016811534762382507, + "fcm_dpo/margin": 245.9033966064453, + "fcm_dpo/q_t": 0.4212663769721985, + "grad_norm": 53.11301803588867, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.724645733833313, + "logits/rejected": -0.7101547122001648, + "logps/chosen": -575.8922119140625, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -857.44189453125, + "loss": 1.1638, + "margin_dpo/margin_mean": 245.90341186523438, + "margin_dpo/margin_std": 454.92730712890625, + "step": 441 + }, + { + "KL/chosen_KL_mean": -440.19891357421875, + "KL/mean": -586.6024169921875, + "KL/rejected_KL_mean": -733.005859375, + "KL/std": 351.505615234375, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.001444188179448247, + "fcm_dpo/delta": -0.08036504685878754, + "fcm_dpo/margin": 292.80694580078125, + "fcm_dpo/q_t": 0.4045270085334778, + "grad_norm": 34.267642974853516, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.6593063473701477, + "logits/rejected": -0.6503518223762512, + "logps/chosen": -487.21014404296875, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -809.545166015625, + "loss": 1.0851, + "margin_dpo/margin_mean": 292.80694580078125, + "margin_dpo/margin_std": 389.67095947265625, + "step": 442 + }, + { + "KL/chosen_KL_mean": -491.76605224609375, + "KL/mean": -585.1986694335938, + "KL/rejected_KL_mean": -678.63134765625, + "KL/std": 355.941650390625, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.00143684190697968, + "fcm_dpo/delta": 0.014845379628241062, + "fcm_dpo/margin": 186.86532592773438, + "fcm_dpo/q_t": 0.4393439292907715, + "grad_norm": 42.27107620239258, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.6870307326316833, + "logits/rejected": -0.6640233993530273, + "logps/chosen": -563.0390625, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -765.311279296875, + "loss": 1.2291, + "margin_dpo/margin_mean": 186.8653106689453, + "margin_dpo/margin_std": 432.3843078613281, + "step": 443 + }, + { + "KL/chosen_KL_mean": -466.0931396484375, + "KL/mean": -615.40185546875, + "KL/rejected_KL_mean": -764.7105102539062, + "KL/std": 366.80718994140625, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.0014270897954702377, + "fcm_dpo/delta": -0.027542727068066597, + "fcm_dpo/margin": 298.61737060546875, + "fcm_dpo/q_t": 0.40343034267425537, + "grad_norm": 30.802732467651367, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.6375446319580078, + "logits/rejected": -0.6471455097198486, + "logps/chosen": -523.3068237304688, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -861.9653930664062, + "loss": 1.0791, + "margin_dpo/margin_mean": 298.6174011230469, + "margin_dpo/margin_std": 400.170654296875, + "step": 444 + }, + { + "KL/chosen_KL_mean": -413.6000061035156, + "KL/mean": -561.503173828125, + "KL/rejected_KL_mean": -709.4063720703125, + "KL/std": 302.09783935546875, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.00142662413418293, + "fcm_dpo/delta": -0.02299850434064865, + "fcm_dpo/margin": 295.806396484375, + "fcm_dpo/q_t": 0.4039134979248047, + "grad_norm": 32.144405364990234, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.6647120714187622, + "logits/rejected": -0.6569557189941406, + "logps/chosen": -480.8997802734375, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -802.0889892578125, + "loss": 1.0767, + "margin_dpo/margin_mean": 295.8063659667969, + "margin_dpo/margin_std": 374.839599609375, + "step": 445 + }, + { + "KL/chosen_KL_mean": -385.03167724609375, + "KL/mean": -539.4615478515625, + "KL/rejected_KL_mean": -693.8914184570312, + "KL/std": 323.5318298339844, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.0014220019802451134, + "fcm_dpo/delta": -0.04146070033311844, + "fcm_dpo/margin": 308.85968017578125, + "fcm_dpo/q_t": 0.39705991744995117, + "grad_norm": 43.11029052734375, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.6826563477516174, + "logits/rejected": -0.6982432007789612, + "logps/chosen": -444.13018798828125, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -795.1555786132812, + "loss": 1.0501, + "margin_dpo/margin_mean": 308.85968017578125, + "margin_dpo/margin_std": 330.52618408203125, + "step": 446 + }, + { + "KL/chosen_KL_mean": -408.96429443359375, + "KL/mean": -570.580078125, + "KL/rejected_KL_mean": -732.19580078125, + "KL/std": 381.03350830078125, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0013954371679574251, + "fcm_dpo/delta": -0.05377761647105217, + "fcm_dpo/margin": 323.2315673828125, + "fcm_dpo/q_t": 0.3988439440727234, + "grad_norm": 27.523120880126953, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.7374973297119141, + "logits/rejected": -0.7542306184768677, + "logps/chosen": -465.03961181640625, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -830.890625, + "loss": 1.0611, + "margin_dpo/margin_mean": 323.2315673828125, + "margin_dpo/margin_std": 423.53790283203125, + "step": 447 + }, + { + "KL/chosen_KL_mean": -442.3367614746094, + "KL/mean": -596.041259765625, + "KL/rejected_KL_mean": -749.7457275390625, + "KL/std": 328.1753234863281, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0013942192308604717, + "fcm_dpo/delta": -0.0300214271992445, + "fcm_dpo/margin": 307.4089660644531, + "fcm_dpo/q_t": 0.4010792672634125, + "grad_norm": 38.13869094848633, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.6571661233901978, + "logits/rejected": -0.6673141717910767, + "logps/chosen": -502.3406066894531, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -852.0103759765625, + "loss": 1.0656, + "margin_dpo/margin_mean": 307.4089660644531, + "margin_dpo/margin_std": 369.6396179199219, + "step": 448 + }, + { + "KL/chosen_KL_mean": -463.1051025390625, + "KL/mean": -620.4819946289062, + "KL/rejected_KL_mean": -777.85888671875, + "KL/std": 402.813720703125, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.0013779536820948124, + "fcm_dpo/delta": -0.03533675894141197, + "fcm_dpo/margin": 314.7537841796875, + "fcm_dpo/q_t": 0.40381747484207153, + "grad_norm": 29.83260726928711, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.6919020414352417, + "logits/rejected": -0.6886130571365356, + "logps/chosen": -530.6317138671875, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -866.455810546875, + "loss": 1.0897, + "margin_dpo/margin_mean": 314.7537841796875, + "margin_dpo/margin_std": 467.1162109375, + "step": 449 + }, + { + "KL/chosen_KL_mean": -376.8255615234375, + "KL/mean": -544.375732421875, + "KL/rejected_KL_mean": -711.9259033203125, + "KL/std": 348.3739318847656, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0013674888759851456, + "fcm_dpo/delta": -0.061000145971775055, + "fcm_dpo/margin": 335.10040283203125, + "fcm_dpo/q_t": 0.392261803150177, + "grad_norm": 53.29400634765625, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.7258347868919373, + "logits/rejected": -0.7416701316833496, + "logps/chosen": -433.93365478515625, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -814.680908203125, + "loss": 1.031, + "margin_dpo/margin_mean": 335.10040283203125, + "margin_dpo/margin_std": 342.30657958984375, + "step": 450 + }, + { + "KL/chosen_KL_mean": -515.3168334960938, + "KL/mean": -635.5189208984375, + "KL/rejected_KL_mean": -755.7210693359375, + "KL/std": 415.734619140625, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0013626276049762964, + "fcm_dpo/delta": 0.013939025811851025, + "fcm_dpo/margin": 240.40431213378906, + "fcm_dpo/q_t": 0.4219001531600952, + "grad_norm": 43.0428581237793, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.7938821315765381, + "logits/rejected": -0.7706050872802734, + "logps/chosen": -573.78564453125, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -828.6505126953125, + "loss": 1.1878, + "margin_dpo/margin_mean": 240.40432739257812, + "margin_dpo/margin_std": 487.4724426269531, + "step": 451 + }, + { + "KL/chosen_KL_mean": -364.3936767578125, + "KL/mean": -511.3393249511719, + "KL/rejected_KL_mean": -658.284912109375, + "KL/std": 325.40350341796875, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.0013630865141749382, + "fcm_dpo/delta": -0.0006436249241232872, + "fcm_dpo/margin": 293.8912048339844, + "fcm_dpo/q_t": 0.4053363800048828, + "grad_norm": 20.500471115112305, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.6958132982254028, + "logits/rejected": -0.7123762965202332, + "logps/chosen": -414.78424072265625, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -756.0563354492188, + "loss": 1.0701, + "margin_dpo/margin_mean": 293.8912353515625, + "margin_dpo/margin_std": 318.0101318359375, + "step": 452 + }, + { + "KL/chosen_KL_mean": -422.84063720703125, + "KL/mean": -566.01904296875, + "KL/rejected_KL_mean": -709.1974487304688, + "KL/std": 326.71624755859375, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0013635011855512857, + "fcm_dpo/delta": 0.009871179237961769, + "fcm_dpo/margin": 286.3568115234375, + "fcm_dpo/q_t": 0.4116683006286621, + "grad_norm": 32.41584777832031, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.7237302660942078, + "logits/rejected": -0.712169885635376, + "logps/chosen": -480.55548095703125, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -791.4049072265625, + "loss": 1.1, + "margin_dpo/margin_mean": 286.3568115234375, + "margin_dpo/margin_std": 396.763427734375, + "step": 453 + }, + { + "KL/chosen_KL_mean": -515.9754638671875, + "KL/mean": -676.83642578125, + "KL/rejected_KL_mean": -837.697509765625, + "KL/std": 378.63494873046875, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0013557194033637643, + "fcm_dpo/delta": -0.0380592904984951, + "fcm_dpo/margin": 321.7220153808594, + "fcm_dpo/q_t": 0.400535523891449, + "grad_norm": 27.571773529052734, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.7779266834259033, + "logits/rejected": -0.7719443440437317, + "logps/chosen": -576.9210815429688, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -922.6482543945312, + "loss": 1.0692, + "margin_dpo/margin_mean": 321.7220153808594, + "margin_dpo/margin_std": 418.5616760253906, + "step": 454 + }, + { + "KL/chosen_KL_mean": -456.84912109375, + "KL/mean": -642.4649047851562, + "KL/rejected_KL_mean": -828.0806884765625, + "KL/std": 389.8719177246094, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0013439783360809088, + "fcm_dpo/delta": -0.10423971712589264, + "fcm_dpo/margin": 371.2315979003906, + "fcm_dpo/q_t": 0.3871069848537445, + "grad_norm": 38.229557037353516, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.7495969533920288, + "logits/rejected": -0.7755312323570251, + "logps/chosen": -501.7358093261719, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -943.3822021484375, + "loss": 1.0312, + "margin_dpo/margin_mean": 371.2315979003906, + "margin_dpo/margin_std": 439.0977783203125, + "step": 455 + }, + { + "KL/chosen_KL_mean": -464.5408935546875, + "KL/mean": -646.1202392578125, + "KL/rejected_KL_mean": -827.69970703125, + "KL/std": 395.7155456542969, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.001314440043643117, + "fcm_dpo/delta": -0.0811576098203659, + "fcm_dpo/margin": 363.1587829589844, + "fcm_dpo/q_t": 0.39027196168899536, + "grad_norm": 40.24952697753906, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.7377710342407227, + "logits/rejected": -0.7650057077407837, + "logps/chosen": -521.57763671875, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -932.91748046875, + "loss": 1.0204, + "margin_dpo/margin_mean": 363.1587829589844, + "margin_dpo/margin_std": 379.25030517578125, + "step": 456 + }, + { + "KL/chosen_KL_mean": -460.05450439453125, + "KL/mean": -644.2169189453125, + "KL/rejected_KL_mean": -828.3792724609375, + "KL/std": 388.38189697265625, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.0012894996907562017, + "fcm_dpo/delta": -0.07892660796642303, + "fcm_dpo/margin": 368.32470703125, + "fcm_dpo/q_t": 0.3893137276172638, + "grad_norm": 30.760709762573242, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.7626906633377075, + "logits/rejected": -0.7672203183174133, + "logps/chosen": -514.2970581054688, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -913.4888305664062, + "loss": 1.0235, + "margin_dpo/margin_mean": 368.32470703125, + "margin_dpo/margin_std": 381.91180419921875, + "step": 457 + }, + { + "KL/chosen_KL_mean": -504.82781982421875, + "KL/mean": -659.931640625, + "KL/rejected_KL_mean": -815.0355224609375, + "KL/std": 357.541015625, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.0012719187652692199, + "fcm_dpo/delta": -0.05825035274028778, + "fcm_dpo/margin": 310.2076721191406, + "fcm_dpo/q_t": 0.4102453589439392, + "grad_norm": 40.47724533081055, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.7630763649940491, + "logits/rejected": -0.7581685185432434, + "logps/chosen": -560.2366943359375, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -912.71875, + "loss": 1.0951, + "margin_dpo/margin_mean": 310.2076721191406, + "margin_dpo/margin_std": 405.93267822265625, + "step": 458 + }, + { + "KL/chosen_KL_mean": -539.5087890625, + "KL/mean": -724.222900390625, + "KL/rejected_KL_mean": -908.9368896484375, + "KL/std": 430.0087890625, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.001254374161362648, + "fcm_dpo/delta": -0.06705733388662338, + "fcm_dpo/margin": 369.4280700683594, + "fcm_dpo/q_t": 0.3950594663619995, + "grad_norm": 29.82525062561035, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.8026560544967651, + "logits/rejected": -0.8276393413543701, + "logps/chosen": -586.0662841796875, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -995.10546875, + "loss": 1.0606, + "margin_dpo/margin_mean": 369.4281005859375, + "margin_dpo/margin_std": 480.92803955078125, + "step": 459 + }, + { + "KL/chosen_KL_mean": -588.6797485351562, + "KL/mean": -787.8248901367188, + "KL/rejected_KL_mean": -986.969970703125, + "KL/std": 426.8436584472656, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0012380112893879414, + "fcm_dpo/delta": -0.09784276783466339, + "fcm_dpo/margin": 398.2901916503906, + "fcm_dpo/q_t": 0.3881361484527588, + "grad_norm": 35.1406135559082, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.8180075883865356, + "logits/rejected": -0.8504735827445984, + "logps/chosen": -640.314697265625, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -1091.08935546875, + "loss": 1.0253, + "margin_dpo/margin_mean": 398.2901611328125, + "margin_dpo/margin_std": 460.20306396484375, + "step": 460 + }, + { + "KL/chosen_KL_mean": -628.321533203125, + "KL/mean": -779.9276123046875, + "KL/rejected_KL_mean": -931.53369140625, + "KL/std": 414.652099609375, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.001238570548593998, + "fcm_dpo/delta": 0.025193795561790466, + "fcm_dpo/margin": 303.21209716796875, + "fcm_dpo/q_t": 0.4151153564453125, + "grad_norm": 38.090084075927734, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.8387245535850525, + "logits/rejected": -0.8543214797973633, + "logps/chosen": -683.5034790039062, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1018.0105590820312, + "loss": 1.1147, + "margin_dpo/margin_mean": 303.21209716796875, + "margin_dpo/margin_std": 442.78985595703125, + "step": 461 + }, + { + "KL/chosen_KL_mean": -597.3856201171875, + "KL/mean": -727.8646240234375, + "KL/rejected_KL_mean": -858.343505859375, + "KL/std": 418.62811279296875, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.001244060229510069, + "fcm_dpo/delta": 0.028975196182727814, + "fcm_dpo/margin": 260.9578857421875, + "fcm_dpo/q_t": 0.42770883440971375, + "grad_norm": 37.44738006591797, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.903153121471405, + "logits/rejected": -0.894717812538147, + "logps/chosen": -667.3136596679688, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -937.1846923828125, + "loss": 1.1666, + "margin_dpo/margin_mean": 260.9578857421875, + "margin_dpo/margin_std": 469.716796875, + "step": 462 + }, + { + "KL/chosen_KL_mean": -610.7987060546875, + "KL/mean": -781.158447265625, + "KL/rejected_KL_mean": -951.5181884765625, + "KL/std": 427.88287353515625, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.0012440024875104427, + "fcm_dpo/delta": -0.02509607933461666, + "fcm_dpo/margin": 340.719482421875, + "fcm_dpo/q_t": 0.4041438698768616, + "grad_norm": 47.5179557800293, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.8641135096549988, + "logits/rejected": -0.8778239488601685, + "logps/chosen": -666.0731201171875, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -1040.543212890625, + "loss": 1.0856, + "margin_dpo/margin_mean": 340.7194519042969, + "margin_dpo/margin_std": 467.81512451171875, + "step": 463 + }, + { + "KL/chosen_KL_mean": -584.25732421875, + "KL/mean": -836.221435546875, + "KL/rejected_KL_mean": -1088.185546875, + "KL/std": 492.1297912597656, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0011908018495887518, + "fcm_dpo/delta": -0.215261772274971, + "fcm_dpo/margin": 503.92816162109375, + "fcm_dpo/q_t": 0.36349034309387207, + "grad_norm": 45.95230484008789, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.8583030700683594, + "logits/rejected": -0.9283461570739746, + "logps/chosen": -635.169677734375, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1190.6749267578125, + "loss": 0.9527, + "margin_dpo/margin_mean": 503.92816162109375, + "margin_dpo/margin_std": 500.1207275390625, + "step": 464 + }, + { + "KL/chosen_KL_mean": -582.9863891601562, + "KL/mean": -786.6162109375, + "KL/rejected_KL_mean": -990.2459716796875, + "KL/std": 502.50006103515625, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.0011753500439226627, + "fcm_dpo/delta": -0.08259066939353943, + "fcm_dpo/margin": 407.2596435546875, + "fcm_dpo/q_t": 0.3880874216556549, + "grad_norm": 32.566383361816406, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.8125755786895752, + "logits/rejected": -0.8418079614639282, + "logps/chosen": -643.103271484375, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -1104.1920166015625, + "loss": 1.0509, + "margin_dpo/margin_mean": 407.2596435546875, + "margin_dpo/margin_std": 516.5306396484375, + "step": 465 + }, + { + "KL/chosen_KL_mean": -639.4822998046875, + "KL/mean": -815.3917236328125, + "KL/rejected_KL_mean": -991.301025390625, + "KL/std": 460.76763916015625, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0011590267531573772, + "fcm_dpo/delta": -0.008637342602014542, + "fcm_dpo/margin": 351.81878662109375, + "fcm_dpo/q_t": 0.406246155500412, + "grad_norm": 39.65568542480469, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.846731960773468, + "logits/rejected": -0.8508492708206177, + "logps/chosen": -692.4031982421875, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -1081.616455078125, + "loss": 1.1002, + "margin_dpo/margin_mean": 351.8188171386719, + "margin_dpo/margin_std": 497.9888916015625, + "step": 466 + }, + { + "KL/chosen_KL_mean": -795.7896728515625, + "KL/mean": -982.810791015625, + "KL/rejected_KL_mean": -1169.83203125, + "KL/std": 595.2113037109375, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0011420733062550426, + "fcm_dpo/delta": -0.09609047323465347, + "fcm_dpo/margin": 374.0423583984375, + "fcm_dpo/q_t": 0.40663576126098633, + "grad_norm": 67.05147552490234, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.9450754523277283, + "logits/rejected": -0.9458522796630859, + "logps/chosen": -874.5054931640625, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1272.692138671875, + "loss": 1.1508, + "margin_dpo/margin_mean": 374.0423583984375, + "margin_dpo/margin_std": 697.7740478515625, + "step": 467 + }, + { + "KL/chosen_KL_mean": -604.61865234375, + "KL/mean": -829.852294921875, + "KL/rejected_KL_mean": -1055.0859375, + "KL/std": 519.4028930664062, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.001122980727814138, + "fcm_dpo/delta": -0.11154043674468994, + "fcm_dpo/margin": 450.46722412109375, + "fcm_dpo/q_t": 0.3880060613155365, + "grad_norm": 80.06796264648438, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.8348629474639893, + "logits/rejected": -0.8475467562675476, + "logps/chosen": -674.5540771484375, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1156.11474609375, + "loss": 1.0331, + "margin_dpo/margin_mean": 450.46722412109375, + "margin_dpo/margin_std": 576.6068115234375, + "step": 468 + }, + { + "KL/chosen_KL_mean": -625.6953125, + "KL/mean": -855.228271484375, + "KL/rejected_KL_mean": -1084.76123046875, + "KL/std": 460.6417236328125, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.001093520550057292, + "fcm_dpo/delta": -0.10844644904136658, + "fcm_dpo/margin": 459.06585693359375, + "fcm_dpo/q_t": 0.3849395513534546, + "grad_norm": 48.40918731689453, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.8724262714385986, + "logits/rejected": -0.8904796838760376, + "logps/chosen": -693.820068359375, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1189.547607421875, + "loss": 1.0199, + "margin_dpo/margin_mean": 459.06585693359375, + "margin_dpo/margin_std": 515.1187133789062, + "step": 469 + }, + { + "KL/chosen_KL_mean": -520.3056640625, + "KL/mean": -713.55810546875, + "KL/rejected_KL_mean": -906.810546875, + "KL/std": 473.5939636230469, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.001092412043362856, + "fcm_dpo/delta": -0.023316729813814163, + "fcm_dpo/margin": 386.50494384765625, + "fcm_dpo/q_t": 0.402747243642807, + "grad_norm": 31.65645408630371, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.8062844276428223, + "logits/rejected": -0.8211459517478943, + "logps/chosen": -564.0975341796875, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -989.513427734375, + "loss": 1.0861, + "margin_dpo/margin_mean": 386.5049743652344, + "margin_dpo/margin_std": 528.432861328125, + "step": 470 + }, + { + "KL/chosen_KL_mean": -669.3486328125, + "KL/mean": -839.7194213867188, + "KL/rejected_KL_mean": -1010.0902099609375, + "KL/std": 498.46844482421875, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.001080801011994481, + "fcm_dpo/delta": -0.03241042047739029, + "fcm_dpo/margin": 340.74151611328125, + "fcm_dpo/q_t": 0.41777703166007996, + "grad_norm": 41.95170974731445, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.9357224702835083, + "logits/rejected": -0.9499038457870483, + "logps/chosen": -732.6881713867188, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1093.70068359375, + "loss": 1.1305, + "margin_dpo/margin_mean": 340.74151611328125, + "margin_dpo/margin_std": 542.1281127929688, + "step": 471 + }, + { + "KL/chosen_KL_mean": -683.3143310546875, + "KL/mean": -881.6566162109375, + "KL/rejected_KL_mean": -1079.998779296875, + "KL/std": 597.828857421875, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.0010728872148320079, + "fcm_dpo/delta": -0.02705109678208828, + "fcm_dpo/margin": 396.6844482421875, + "fcm_dpo/q_t": 0.4087257981300354, + "grad_norm": 39.79328155517578, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.9523841738700867, + "logits/rejected": -0.9378076791763306, + "logps/chosen": -766.98046875, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1197.2080078125, + "loss": 1.1192, + "margin_dpo/margin_mean": 396.6844177246094, + "margin_dpo/margin_std": 654.9652709960938, + "step": 472 + }, + { + "KL/chosen_KL_mean": -813.05029296875, + "KL/mean": -903.286376953125, + "KL/rejected_KL_mean": -993.5223999023438, + "KL/std": 609.0579223632812, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0010749694192782044, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 180.4720458984375, + "fcm_dpo/q_t": 0.4577036499977112, + "grad_norm": 158.85025024414062, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.9087912440299988, + "logits/rejected": -0.885597288608551, + "logps/chosen": -876.5473022460938, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1074.6689453125, + "loss": 1.3599, + "margin_dpo/margin_mean": 180.4720458984375, + "margin_dpo/margin_std": 793.6195068359375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -684.17578125, + "KL/mean": -856.5277709960938, + "KL/rejected_KL_mean": -1028.879638671875, + "KL/std": 539.4508666992188, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.0010672679636627436, + "fcm_dpo/delta": -0.07216163724660873, + "fcm_dpo/margin": 344.7039794921875, + "fcm_dpo/q_t": 0.41513219475746155, + "grad_norm": 40.7275505065918, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.879541277885437, + "logits/rejected": -0.8852543830871582, + "logps/chosen": -736.7877197265625, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1118.9600830078125, + "loss": 1.1576, + "margin_dpo/margin_mean": 344.70391845703125, + "margin_dpo/margin_std": 607.1236572265625, + "step": 474 + }, + { + "KL/chosen_KL_mean": -499.65985107421875, + "KL/mean": -718.20947265625, + "KL/rejected_KL_mean": -936.759033203125, + "KL/std": 435.0946350097656, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0010481432545930147, + "fcm_dpo/delta": -0.06110334023833275, + "fcm_dpo/margin": 437.09912109375, + "fcm_dpo/q_t": 0.3936186730861664, + "grad_norm": 53.17040252685547, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.9238021969795227, + "logits/rejected": -0.949226438999176, + "logps/chosen": -542.155029296875, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -1026.822021484375, + "loss": 1.0374, + "margin_dpo/margin_mean": 437.09912109375, + "margin_dpo/margin_std": 474.75665283203125, + "step": 475 + }, + { + "KL/chosen_KL_mean": -613.9573974609375, + "KL/mean": -793.204833984375, + "KL/rejected_KL_mean": -972.4522705078125, + "KL/std": 495.7106018066406, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.001049616257660091, + "fcm_dpo/delta": 0.024651065468788147, + "fcm_dpo/margin": 358.494873046875, + "fcm_dpo/q_t": 0.41439297795295715, + "grad_norm": 75.15031433105469, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.9037166833877563, + "logits/rejected": -0.9253528118133545, + "logps/chosen": -656.90673828125, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1046.16259765625, + "loss": 1.1064, + "margin_dpo/margin_mean": 358.494873046875, + "margin_dpo/margin_std": 493.5234069824219, + "step": 476 + }, + { + "KL/chosen_KL_mean": -666.6996459960938, + "KL/mean": -844.0084228515625, + "KL/rejected_KL_mean": -1021.3171997070312, + "KL/std": 521.0535278320312, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0010592498583719134, + "fcm_dpo/delta": 0.025024261325597763, + "fcm_dpo/margin": 354.6175842285156, + "fcm_dpo/q_t": 0.4148496389389038, + "grad_norm": 36.149314880371094, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.9922509789466858, + "logits/rejected": -0.9685536623001099, + "logps/chosen": -737.4722900390625, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -1097.45458984375, + "loss": 1.1452, + "margin_dpo/margin_mean": 354.6175842285156, + "margin_dpo/margin_std": 620.2078857421875, + "step": 477 + }, + { + "KL/chosen_KL_mean": -559.6727905273438, + "KL/mean": -770.5025634765625, + "KL/rejected_KL_mean": -981.3324584960938, + "KL/std": 474.78558349609375, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.0010527204722166061, + "fcm_dpo/delta": -0.045949313789606094, + "fcm_dpo/margin": 421.6596984863281, + "fcm_dpo/q_t": 0.39920175075531006, + "grad_norm": 46.123207092285156, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.8505024909973145, + "logits/rejected": -0.8774957656860352, + "logps/chosen": -601.11328125, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -1066.6944580078125, + "loss": 1.0701, + "margin_dpo/margin_mean": 421.65972900390625, + "margin_dpo/margin_std": 557.5150146484375, + "step": 478 + }, + { + "KL/chosen_KL_mean": -732.49267578125, + "KL/mean": -932.5208740234375, + "KL/rejected_KL_mean": -1132.549072265625, + "KL/std": 584.312255859375, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0010395834688097239, + "fcm_dpo/delta": -0.07593031227588654, + "fcm_dpo/margin": 400.05645751953125, + "fcm_dpo/q_t": 0.40925589203834534, + "grad_norm": 36.56541061401367, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.9467741250991821, + "logits/rejected": -0.9780069589614868, + "logps/chosen": -786.400634765625, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1227.66552734375, + "loss": 1.1247, + "margin_dpo/margin_mean": 400.05645751953125, + "margin_dpo/margin_std": 675.3660888671875, + "step": 479 + }, + { + "KL/chosen_KL_mean": -866.382568359375, + "KL/mean": -1019.0773315429688, + "KL/rejected_KL_mean": -1171.7720947265625, + "KL/std": 550.42041015625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0010277291294187307, + "fcm_dpo/delta": -0.03854019567370415, + "fcm_dpo/margin": 305.3895263671875, + "fcm_dpo/q_t": 0.429998517036438, + "grad_norm": 66.87673950195312, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.9413371086120605, + "logits/rejected": -0.9438562393188477, + "logps/chosen": -925.0653076171875, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1254.70458984375, + "loss": 1.196, + "margin_dpo/margin_mean": 305.3895263671875, + "margin_dpo/margin_std": 602.7835693359375, + "step": 480 + }, + { + "KL/chosen_KL_mean": -696.2211303710938, + "KL/mean": -935.7109985351562, + "KL/rejected_KL_mean": -1175.2008056640625, + "KL/std": 561.6510009765625, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.0010110186412930489, + "fcm_dpo/delta": -0.08849819004535675, + "fcm_dpo/margin": 478.9796142578125, + "fcm_dpo/q_t": 0.3909313380718231, + "grad_norm": 34.13664627075195, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.9505438804626465, + "logits/rejected": -0.9816187024116516, + "logps/chosen": -751.1854248046875, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1267.621337890625, + "loss": 1.0431, + "margin_dpo/margin_mean": 478.9796142578125, + "margin_dpo/margin_std": 602.5888061523438, + "step": 481 + }, + { + "KL/chosen_KL_mean": -771.9962158203125, + "KL/mean": -880.0143432617188, + "KL/rejected_KL_mean": -988.032470703125, + "KL/std": 573.8893432617188, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.001005807425826788, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 216.03623962402344, + "fcm_dpo/q_t": 0.4478261172771454, + "grad_norm": 89.22106170654297, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.8619707822799683, + "logits/rejected": -0.8561730980873108, + "logps/chosen": -839.5496826171875, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1075.6220703125, + "loss": 1.2838, + "margin_dpo/margin_mean": 216.0362548828125, + "margin_dpo/margin_std": 664.136474609375, + "step": 482 + }, + { + "KL/chosen_KL_mean": -681.9205932617188, + "KL/mean": -925.708251953125, + "KL/rejected_KL_mean": -1169.495849609375, + "KL/std": 555.2496948242188, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.000990971690043807, + "fcm_dpo/delta": -0.08762480318546295, + "fcm_dpo/margin": 487.5753479003906, + "fcm_dpo/q_t": 0.3924211859703064, + "grad_norm": 38.14697265625, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.9329118728637695, + "logits/rejected": -0.9486919641494751, + "logps/chosen": -745.2503662109375, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1265.282958984375, + "loss": 1.0427, + "margin_dpo/margin_mean": 487.5753173828125, + "margin_dpo/margin_std": 617.0916748046875, + "step": 483 + }, + { + "KL/chosen_KL_mean": -592.4403686523438, + "KL/mean": -823.39794921875, + "KL/rejected_KL_mean": -1054.3555908203125, + "KL/std": 550.065673828125, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0009839143604040146, + "fcm_dpo/delta": -0.05717844516038895, + "fcm_dpo/margin": 461.91522216796875, + "fcm_dpo/q_t": 0.397319495677948, + "grad_norm": 61.332298278808594, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -0.9555022120475769, + "logits/rejected": -0.9872146844863892, + "logps/chosen": -651.5784912109375, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1138.72705078125, + "loss": 1.0562, + "margin_dpo/margin_mean": 461.91522216796875, + "margin_dpo/margin_std": 574.1728515625, + "step": 484 + }, + { + "KL/chosen_KL_mean": -597.3248291015625, + "KL/mean": -817.4119262695312, + "KL/rejected_KL_mean": -1037.4991455078125, + "KL/std": 534.2381591796875, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0009750367607921362, + "fcm_dpo/delta": -0.030541367828845978, + "fcm_dpo/margin": 440.17425537109375, + "fcm_dpo/q_t": 0.40336543321609497, + "grad_norm": 31.684206008911133, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.8777140378952026, + "logits/rejected": -0.9051682949066162, + "logps/chosen": -656.17431640625, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1140.8631591796875, + "loss": 1.0826, + "margin_dpo/margin_mean": 440.1742248535156, + "margin_dpo/margin_std": 606.7274169921875, + "step": 485 + }, + { + "KL/chosen_KL_mean": -688.5882568359375, + "KL/mean": -930.1815185546875, + "KL/rejected_KL_mean": -1171.77490234375, + "KL/std": 613.0765380859375, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.0009599585318937898, + "fcm_dpo/delta": -0.06710436195135117, + "fcm_dpo/margin": 483.18658447265625, + "fcm_dpo/q_t": 0.3970934748649597, + "grad_norm": 50.028724670410156, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -0.978148341178894, + "logits/rejected": -1.0053927898406982, + "logps/chosen": -743.847900390625, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1263.914306640625, + "loss": 1.0912, + "margin_dpo/margin_mean": 483.18658447265625, + "margin_dpo/margin_std": 748.222900390625, + "step": 486 + }, + { + "KL/chosen_KL_mean": -715.52685546875, + "KL/mean": -892.3511962890625, + "KL/rejected_KL_mean": -1069.175537109375, + "KL/std": 545.237060546875, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0009577693999744952, + "fcm_dpo/delta": -0.004492553416639566, + "fcm_dpo/margin": 353.6487731933594, + "fcm_dpo/q_t": 0.4204182028770447, + "grad_norm": 43.76426696777344, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -0.9564817547798157, + "logits/rejected": -0.9717104434967041, + "logps/chosen": -768.590087890625, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1161.594482421875, + "loss": 1.1408, + "margin_dpo/margin_mean": 353.64874267578125, + "margin_dpo/margin_std": 545.6905517578125, + "step": 487 + }, + { + "KL/chosen_KL_mean": -572.8516845703125, + "KL/mean": -749.779296875, + "KL/rejected_KL_mean": -926.7069091796875, + "KL/std": 475.408935546875, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0009570779511705041, + "fcm_dpo/delta": -0.0027291816659271717, + "fcm_dpo/margin": 353.85516357421875, + "fcm_dpo/q_t": 0.4206276834011078, + "grad_norm": 40.64238739013672, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.9040135741233826, + "logits/rejected": -0.9133492112159729, + "logps/chosen": -625.079833984375, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1010.7135009765625, + "loss": 1.1242, + "margin_dpo/margin_mean": 353.85516357421875, + "margin_dpo/margin_std": 467.00531005859375, + "step": 488 + }, + { + "KL/chosen_KL_mean": -532.3974609375, + "KL/mean": -713.440185546875, + "KL/rejected_KL_mean": -894.4829711914062, + "KL/std": 475.89019775390625, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0009564459323883057, + "fcm_dpo/delta": -0.0038762737531214952, + "fcm_dpo/margin": 362.0854797363281, + "fcm_dpo/q_t": 0.4209138751029968, + "grad_norm": 32.53706741333008, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.9166856408119202, + "logits/rejected": -0.9216014742851257, + "logps/chosen": -588.3870849609375, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -973.881103515625, + "loss": 1.1284, + "margin_dpo/margin_mean": 362.08544921875, + "margin_dpo/margin_std": 525.1748046875, + "step": 489 + }, + { + "KL/chosen_KL_mean": -636.086669921875, + "KL/mean": -819.2425537109375, + "KL/rejected_KL_mean": -1002.3983154296875, + "KL/std": 573.73193359375, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0009561643237248063, + "fcm_dpo/delta": 0.0009316356736235321, + "fcm_dpo/margin": 366.3116149902344, + "fcm_dpo/q_t": 0.42113253474235535, + "grad_norm": 41.01726531982422, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.9445546865463257, + "logits/rejected": -0.9854705333709717, + "logps/chosen": -688.453125, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1112.807373046875, + "loss": 1.1605, + "margin_dpo/margin_mean": 366.3116149902344, + "margin_dpo/margin_std": 665.4207763671875, + "step": 490 + }, + { + "KL/chosen_KL_mean": -590.6246337890625, + "KL/mean": -712.3672485351562, + "KL/rejected_KL_mean": -834.10986328125, + "KL/std": 469.73809814453125, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0009562533814460039, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 243.48529052734375, + "fcm_dpo/q_t": 0.44551825523376465, + "grad_norm": 28.462425231933594, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.9519343376159668, + "logits/rejected": -0.9508737921714783, + "logps/chosen": -650.7408447265625, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -907.3826904296875, + "loss": 1.216, + "margin_dpo/margin_mean": 243.4853057861328, + "margin_dpo/margin_std": 473.91986083984375, + "step": 491 + }, + { + "KL/chosen_KL_mean": -604.648193359375, + "KL/mean": -729.9956665039062, + "KL/rejected_KL_mean": -855.3432006835938, + "KL/std": 491.4861145019531, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0009562533814460039, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 250.6949462890625, + "fcm_dpo/q_t": 0.44443410634994507, + "grad_norm": 33.94939041137695, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -0.9876843690872192, + "logits/rejected": -0.9690656065940857, + "logps/chosen": -658.6423950195312, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -928.0028076171875, + "loss": 1.2279, + "margin_dpo/margin_mean": 250.69497680664062, + "margin_dpo/margin_std": 551.9810791015625, + "step": 492 + }, + { + "KL/chosen_KL_mean": -656.4905395507812, + "KL/mean": -797.9208984375, + "KL/rejected_KL_mean": -939.3511352539062, + "KL/std": 548.1793212890625, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0009577979799360037, + "fcm_dpo/delta": 0.008069749921560287, + "fcm_dpo/margin": 282.860595703125, + "fcm_dpo/q_t": 0.4359271228313446, + "grad_norm": 44.11776351928711, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -1.0224084854125977, + "logits/rejected": -0.9967280030250549, + "logps/chosen": -731.98779296875, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1026.6741943359375, + "loss": 1.1981, + "margin_dpo/margin_mean": 282.860595703125, + "margin_dpo/margin_std": 534.3634643554688, + "step": 493 + }, + { + "KL/chosen_KL_mean": -524.4808349609375, + "KL/mean": -755.5260009765625, + "KL/rejected_KL_mean": -986.5711669921875, + "KL/std": 495.45391845703125, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.0009519358281977475, + "fcm_dpo/delta": -0.04171246290206909, + "fcm_dpo/margin": 462.0902404785156, + "fcm_dpo/q_t": 0.3969936668872833, + "grad_norm": 54.5218391418457, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.884068489074707, + "logits/rejected": -0.9211517572402954, + "logps/chosen": -565.840087890625, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1072.6624755859375, + "loss": 1.0459, + "margin_dpo/margin_mean": 462.0902404785156, + "margin_dpo/margin_std": 492.6499938964844, + "step": 494 + }, + { + "KL/chosen_KL_mean": -619.27880859375, + "KL/mean": -813.8908081054688, + "KL/rejected_KL_mean": -1008.5028076171875, + "KL/std": 514.2123413085938, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.000955577939748764, + "fcm_dpo/delta": 0.029042836278676987, + "fcm_dpo/margin": 389.2239074707031, + "fcm_dpo/q_t": 0.416611909866333, + "grad_norm": 36.3665885925293, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -0.9927434921264648, + "logits/rejected": -1.002448320388794, + "logps/chosen": -682.8139038085938, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1099.92724609375, + "loss": 1.1264, + "margin_dpo/margin_mean": 389.22393798828125, + "margin_dpo/margin_std": 614.76171875, + "step": 495 + }, + { + "KL/chosen_KL_mean": -760.7084350585938, + "KL/mean": -860.92138671875, + "KL/rejected_KL_mean": -961.13427734375, + "KL/std": 412.16534423828125, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0009553740965202451, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 200.4258270263672, + "fcm_dpo/q_t": 0.4542078375816345, + "grad_norm": 81.97247314453125, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -0.9654371738433838, + "logits/rejected": -0.937667727470398, + "logps/chosen": -833.3004150390625, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1045.463623046875, + "loss": 1.2533, + "margin_dpo/margin_mean": 200.42581176757812, + "margin_dpo/margin_std": 467.03759765625, + "step": 496 + }, + { + "KL/chosen_KL_mean": -705.8157958984375, + "KL/mean": -812.168701171875, + "KL/rejected_KL_mean": -918.5216064453125, + "KL/std": 510.22918701171875, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.000964211008977145, + "fcm_dpo/delta": 0.046035755425691605, + "fcm_dpo/margin": 212.70584106445312, + "fcm_dpo/q_t": 0.4520183801651001, + "grad_norm": 31.424152374267578, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.943501353263855, + "logits/rejected": -0.9321285486221313, + "logps/chosen": -764.4097290039062, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -994.8099365234375, + "loss": 1.2434, + "margin_dpo/margin_mean": 212.70584106445312, + "margin_dpo/margin_std": 466.66632080078125, + "step": 497 + }, + { + "KL/chosen_KL_mean": -692.849365234375, + "KL/mean": -868.648681640625, + "KL/rejected_KL_mean": -1044.4481201171875, + "KL/std": 535.211669921875, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0009724101983010769, + "fcm_dpo/delta": 0.060184068977832794, + "fcm_dpo/margin": 351.59881591796875, + "fcm_dpo/q_t": 0.4211399555206299, + "grad_norm": 38.71786117553711, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.9622161388397217, + "logits/rejected": -0.9510927200317383, + "logps/chosen": -764.0549926757812, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1128.40625, + "loss": 1.1583, + "margin_dpo/margin_mean": 351.59881591796875, + "margin_dpo/margin_std": 619.1399536132812, + "step": 498 + }, + { + "KL/chosen_KL_mean": -619.5465087890625, + "KL/mean": -850.604736328125, + "KL/rejected_KL_mean": -1081.6630859375, + "KL/std": 568.1401977539062, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.000967134372331202, + "fcm_dpo/delta": -0.04926443099975586, + "fcm_dpo/margin": 462.1165771484375, + "fcm_dpo/q_t": 0.398719847202301, + "grad_norm": 42.0836296081543, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.9825119972229004, + "logits/rejected": -1.021845817565918, + "logps/chosen": -670.8016357421875, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1182.74169921875, + "loss": 1.0738, + "margin_dpo/margin_mean": 462.1165771484375, + "margin_dpo/margin_std": 634.3101806640625, + "step": 499 + }, + { + "KL/chosen_KL_mean": -716.7772216796875, + "KL/mean": -907.3453979492188, + "KL/rejected_KL_mean": -1097.91357421875, + "KL/std": 486.9537658691406, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0009698225767351687, + "fcm_dpo/delta": 0.03154091536998749, + "fcm_dpo/margin": 381.136474609375, + "fcm_dpo/q_t": 0.41535496711730957, + "grad_norm": 30.995935440063477, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.9684487581253052, + "logits/rejected": -1.0029065608978271, + "logps/chosen": -773.8046264648438, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1191.847900390625, + "loss": 1.1342, + "margin_dpo/margin_mean": 381.136474609375, + "margin_dpo/margin_std": 609.620849609375, + "step": 500 + }, + { + "KL/chosen_KL_mean": -647.182373046875, + "KL/mean": -846.9375, + "KL/rejected_KL_mean": -1046.692626953125, + "KL/std": 525.1235961914062, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0009757456136867404, + "fcm_dpo/delta": 0.010504677891731262, + "fcm_dpo/margin": 399.51025390625, + "fcm_dpo/q_t": 0.4120762050151825, + "grad_norm": 41.63288116455078, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.9404511451721191, + "logits/rejected": -0.9300814270973206, + "logps/chosen": -701.5418701171875, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1126.849365234375, + "loss": 1.1293, + "margin_dpo/margin_mean": 399.51025390625, + "margin_dpo/margin_std": 657.058349609375, + "step": 501 + }, + { + "KL/chosen_KL_mean": -594.4793701171875, + "KL/mean": -787.625732421875, + "KL/rejected_KL_mean": -980.7720947265625, + "KL/std": 474.49114990234375, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0009785594884306192, + "fcm_dpo/delta": 0.02282343991100788, + "fcm_dpo/margin": 386.292724609375, + "fcm_dpo/q_t": 0.4135451912879944, + "grad_norm": 39.441287994384766, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.8614951372146606, + "logits/rejected": -0.8489676117897034, + "logps/chosen": -662.079833984375, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1063.720947265625, + "loss": 1.0993, + "margin_dpo/margin_mean": 386.292724609375, + "margin_dpo/margin_std": 503.38922119140625, + "step": 502 + }, + { + "KL/chosen_KL_mean": -578.9581298828125, + "KL/mean": -763.1044311523438, + "KL/rejected_KL_mean": -947.250732421875, + "KL/std": 466.3544616699219, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.000981970108114183, + "fcm_dpo/delta": 0.0397348515689373, + "fcm_dpo/margin": 368.2926025390625, + "fcm_dpo/q_t": 0.41873034834861755, + "grad_norm": 26.599992752075195, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.8834594488143921, + "logits/rejected": -0.8785322904586792, + "logps/chosen": -634.0364990234375, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -1029.756103515625, + "loss": 1.1202, + "margin_dpo/margin_mean": 368.2926025390625, + "margin_dpo/margin_std": 536.4227905273438, + "step": 503 + }, + { + "KL/chosen_KL_mean": -626.6210327148438, + "KL/mean": -842.3348999023438, + "KL/rejected_KL_mean": -1058.048828125, + "KL/std": 538.9749145507812, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.000984931830316782, + "fcm_dpo/delta": -0.026064716279506683, + "fcm_dpo/margin": 431.42779541015625, + "fcm_dpo/q_t": 0.4037286043167114, + "grad_norm": 28.118371963500977, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.9255489110946655, + "logits/rejected": -0.9525940418243408, + "logps/chosen": -686.5867919921875, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1161.8109130859375, + "loss": 1.0889, + "margin_dpo/margin_mean": 431.42779541015625, + "margin_dpo/margin_std": 596.4122924804688, + "step": 504 + }, + { + "KL/chosen_KL_mean": -683.4052734375, + "KL/mean": -792.8162231445312, + "KL/rejected_KL_mean": -902.2271728515625, + "KL/std": 490.6641845703125, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.000981617602519691, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 218.82192993164062, + "fcm_dpo/q_t": 0.44951409101486206, + "grad_norm": 39.305511474609375, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.9343281984329224, + "logits/rejected": -0.9109665155410767, + "logps/chosen": -759.5601806640625, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -990.8125610351562, + "loss": 1.2386, + "margin_dpo/margin_mean": 218.82192993164062, + "margin_dpo/margin_std": 482.3436279296875, + "step": 505 + }, + { + "KL/chosen_KL_mean": -551.6368408203125, + "KL/mean": -741.4173583984375, + "KL/rejected_KL_mean": -931.1978759765625, + "KL/std": 500.6394958496094, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.0009771259501576424, + "fcm_dpo/delta": -0.022931430488824844, + "fcm_dpo/margin": 379.5610046386719, + "fcm_dpo/q_t": 0.41416212916374207, + "grad_norm": 43.389808654785156, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.9206233620643616, + "logits/rejected": -0.9396103620529175, + "logps/chosen": -600.597412109375, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -1009.6129150390625, + "loss": 1.1172, + "margin_dpo/margin_mean": 379.56103515625, + "margin_dpo/margin_std": 545.9964599609375, + "step": 506 + }, + { + "KL/chosen_KL_mean": -629.695068359375, + "KL/mean": -801.294189453125, + "KL/rejected_KL_mean": -972.8934326171875, + "KL/std": 576.819091796875, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0009857050608843565, + "fcm_dpo/delta": 0.063906230032444, + "fcm_dpo/margin": 343.19842529296875, + "fcm_dpo/q_t": 0.42598384618759155, + "grad_norm": 31.065458297729492, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.9137831926345825, + "logits/rejected": -0.9211083054542542, + "logps/chosen": -696.4965209960938, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1068.266357421875, + "loss": 1.1627, + "margin_dpo/margin_mean": 343.19842529296875, + "margin_dpo/margin_std": 630.072265625, + "step": 507 + }, + { + "KL/chosen_KL_mean": -655.5257568359375, + "KL/mean": -802.859130859375, + "KL/rejected_KL_mean": -950.192626953125, + "KL/std": 540.3722534179688, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0009933258406817913, + "fcm_dpo/delta": 0.03655217960476875, + "fcm_dpo/margin": 294.66693115234375, + "fcm_dpo/q_t": 0.43464744091033936, + "grad_norm": 37.10373306274414, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.9039261341094971, + "logits/rejected": -0.9088079333305359, + "logps/chosen": -726.8292236328125, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1045.8201904296875, + "loss": 1.2073, + "margin_dpo/margin_mean": 294.66693115234375, + "margin_dpo/margin_std": 631.143310546875, + "step": 508 + }, + { + "KL/chosen_KL_mean": -499.2848205566406, + "KL/mean": -650.5438232421875, + "KL/rejected_KL_mean": -801.8028564453125, + "KL/std": 399.1927490234375, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0009983675554394722, + "fcm_dpo/delta": 0.014130733907222748, + "fcm_dpo/margin": 302.51806640625, + "fcm_dpo/q_t": 0.4304787516593933, + "grad_norm": 31.841365814208984, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.8085857629776001, + "logits/rejected": -0.8064796328544617, + "logps/chosen": -563.103759765625, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -885.059326171875, + "loss": 1.1652, + "margin_dpo/margin_mean": 302.51806640625, + "margin_dpo/margin_std": 493.38250732421875, + "step": 509 + }, + { + "KL/chosen_KL_mean": -606.8471069335938, + "KL/mean": -805.6715087890625, + "KL/rejected_KL_mean": -1004.495849609375, + "KL/std": 450.83428955078125, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0010016015730798244, + "fcm_dpo/delta": 0.00173291377723217, + "fcm_dpo/margin": 397.64874267578125, + "fcm_dpo/q_t": 0.40606942772865295, + "grad_norm": 40.186344146728516, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.8423061370849609, + "logits/rejected": -0.8638745546340942, + "logps/chosen": -658.7255859375, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1107.260986328125, + "loss": 1.0722, + "margin_dpo/margin_mean": 397.64874267578125, + "margin_dpo/margin_std": 433.5397644042969, + "step": 510 + }, + { + "KL/chosen_KL_mean": -573.0216064453125, + "KL/mean": -728.1978759765625, + "KL/rejected_KL_mean": -883.3741455078125, + "KL/std": 486.86669921875, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0010139435762539506, + "fcm_dpo/delta": 0.08803419768810272, + "fcm_dpo/margin": 310.3525085449219, + "fcm_dpo/q_t": 0.42804354429244995, + "grad_norm": 35.14474105834961, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.8950662612915039, + "logits/rejected": -0.9048582315444946, + "logps/chosen": -633.259765625, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -976.23095703125, + "loss": 1.1801, + "margin_dpo/margin_mean": 310.3525085449219, + "margin_dpo/margin_std": 595.3410034179688, + "step": 511 + }, + { + "KL/chosen_KL_mean": -445.17047119140625, + "KL/mean": -662.233642578125, + "KL/rejected_KL_mean": -879.2968139648438, + "KL/std": 441.00921630859375, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.0010103099048137665, + "fcm_dpo/delta": -0.04047447070479393, + "fcm_dpo/margin": 434.1263427734375, + "fcm_dpo/q_t": 0.3973153233528137, + "grad_norm": 68.36514282226562, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.8424865007400513, + "logits/rejected": -0.8679397702217102, + "logps/chosen": -500.07598876953125, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -961.1727294921875, + "loss": 1.043, + "margin_dpo/margin_mean": 434.1263427734375, + "margin_dpo/margin_std": 449.36865234375, + "step": 512 + }, + { + "KL/chosen_KL_mean": -587.7105712890625, + "KL/mean": -735.6759033203125, + "KL/rejected_KL_mean": -883.6412353515625, + "KL/std": 431.00634765625, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.0010087185073643923, + "fcm_dpo/delta": -0.004779556300491095, + "fcm_dpo/margin": 295.9306335449219, + "fcm_dpo/q_t": 0.43110910058021545, + "grad_norm": 49.95344161987305, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.9263660907745361, + "logits/rejected": -0.9163829684257507, + "logps/chosen": -652.6270751953125, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -959.7036743164062, + "loss": 1.1748, + "margin_dpo/margin_mean": 295.9306335449219, + "margin_dpo/margin_std": 507.47076416015625, + "step": 513 + }, + { + "KL/chosen_KL_mean": -583.0838623046875, + "KL/mean": -731.809814453125, + "KL/rejected_KL_mean": -880.53564453125, + "KL/std": 455.3923034667969, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.001011911197565496, + "fcm_dpo/delta": 0.03155136480927467, + "fcm_dpo/margin": 297.4516906738281, + "fcm_dpo/q_t": 0.43169745802879333, + "grad_norm": 37.62335205078125, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.8366047739982605, + "logits/rejected": -0.8305770754814148, + "logps/chosen": -657.3134765625, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -959.481201171875, + "loss": 1.172, + "margin_dpo/margin_mean": 297.4516906738281, + "margin_dpo/margin_std": 518.9459838867188, + "step": 514 + }, + { + "KL/chosen_KL_mean": -491.0048828125, + "KL/mean": -613.4503784179688, + "KL/rejected_KL_mean": -735.8958740234375, + "KL/std": 388.67047119140625, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0010151038877665997, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 244.89096069335938, + "fcm_dpo/q_t": 0.4403025507926941, + "grad_norm": 38.90882873535156, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.8019149303436279, + "logits/rejected": -0.8163158297538757, + "logps/chosen": -541.406494140625, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -822.9935913085938, + "loss": 1.2035, + "margin_dpo/margin_mean": 244.89097595214844, + "margin_dpo/margin_std": 453.50726318359375, + "step": 515 + }, + { + "KL/chosen_KL_mean": -518.7323608398438, + "KL/mean": -695.240478515625, + "KL/rejected_KL_mean": -871.74853515625, + "KL/std": 446.7120361328125, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.00102404470089823, + "fcm_dpo/delta": 0.03965992480516434, + "fcm_dpo/margin": 353.01617431640625, + "fcm_dpo/q_t": 0.4145081639289856, + "grad_norm": 49.82358169555664, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.8736060857772827, + "logits/rejected": -0.9022574424743652, + "logps/chosen": -583.3819580078125, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -983.470947265625, + "loss": 1.1061, + "margin_dpo/margin_mean": 353.01617431640625, + "margin_dpo/margin_std": 438.5834045410156, + "step": 516 + }, + { + "KL/chosen_KL_mean": -524.5887451171875, + "KL/mean": -728.25146484375, + "KL/rejected_KL_mean": -931.9141235351562, + "KL/std": 466.2516784667969, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.001018983544781804, + "fcm_dpo/delta": -0.015801358968019485, + "fcm_dpo/margin": 407.3253173828125, + "fcm_dpo/q_t": 0.4051435589790344, + "grad_norm": 27.777956008911133, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.82435142993927, + "logits/rejected": -0.8253399133682251, + "logps/chosen": -585.5023193359375, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1020.9971923828125, + "loss": 1.0764, + "margin_dpo/margin_mean": 407.3253173828125, + "margin_dpo/margin_std": 514.4728393554688, + "step": 517 + }, + { + "KL/chosen_KL_mean": -483.32427978515625, + "KL/mean": -654.7218017578125, + "KL/rejected_KL_mean": -826.1192626953125, + "KL/std": 386.3504638671875, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.00101938983425498, + "fcm_dpo/delta": -0.002791373757645488, + "fcm_dpo/margin": 342.79498291015625, + "fcm_dpo/q_t": 0.4173525869846344, + "grad_norm": 73.90521240234375, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.8224846124649048, + "logits/rejected": -0.8400793075561523, + "logps/chosen": -540.7801513671875, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -911.4320068359375, + "loss": 1.1141, + "margin_dpo/margin_mean": 342.79498291015625, + "margin_dpo/margin_std": 432.29754638671875, + "step": 518 + }, + { + "KL/chosen_KL_mean": -493.5823059082031, + "KL/mean": -618.7098388671875, + "KL/rejected_KL_mean": -743.8374633789062, + "KL/std": 352.56585693359375, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0010242740390822291, + "fcm_dpo/delta": 0.047684717923402786, + "fcm_dpo/margin": 250.25511169433594, + "fcm_dpo/q_t": 0.4395211338996887, + "grad_norm": 27.285703659057617, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.8396116495132446, + "logits/rejected": -0.8237789869308472, + "logps/chosen": -567.6456298828125, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -848.2816162109375, + "loss": 1.1976, + "margin_dpo/margin_mean": 250.25511169433594, + "margin_dpo/margin_std": 447.3692626953125, + "step": 519 + }, + { + "KL/chosen_KL_mean": -553.9363403320312, + "KL/mean": -707.3574829101562, + "KL/rejected_KL_mean": -860.778564453125, + "KL/std": 420.49395751953125, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.0010321482550352812, + "fcm_dpo/delta": 0.014505407772958279, + "fcm_dpo/margin": 306.8422546386719, + "fcm_dpo/q_t": 0.4273594915866852, + "grad_norm": 39.897335052490234, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.7962254285812378, + "logits/rejected": -0.8004966974258423, + "logps/chosen": -624.2362060546875, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -960.7599487304688, + "loss": 1.1548, + "margin_dpo/margin_mean": 306.8422546386719, + "margin_dpo/margin_std": 473.69488525390625, + "step": 520 + }, + { + "KL/chosen_KL_mean": -475.22137451171875, + "KL/mean": -644.2598266601562, + "KL/rejected_KL_mean": -813.29833984375, + "KL/std": 429.7741394042969, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0010291090002283454, + "fcm_dpo/delta": -0.029532819986343384, + "fcm_dpo/margin": 338.07696533203125, + "fcm_dpo/q_t": 0.41869616508483887, + "grad_norm": 38.53537368774414, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.8567283153533936, + "logits/rejected": -0.8610430955886841, + "logps/chosen": -533.3642578125, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -896.5789184570312, + "loss": 1.1195, + "margin_dpo/margin_mean": 338.07696533203125, + "margin_dpo/margin_std": 440.51312255859375, + "step": 521 + }, + { + "KL/chosen_KL_mean": -548.38818359375, + "KL/mean": -704.2079467773438, + "KL/rejected_KL_mean": -860.0277099609375, + "KL/std": 447.48101806640625, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.0010385368950664997, + "fcm_dpo/delta": 0.07887880504131317, + "fcm_dpo/margin": 311.6395263671875, + "fcm_dpo/q_t": 0.4251254200935364, + "grad_norm": 29.496143341064453, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.8598141670227051, + "logits/rejected": -0.8639324903488159, + "logps/chosen": -594.9358520507812, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -926.0416259765625, + "loss": 1.1496, + "margin_dpo/margin_mean": 311.6395263671875, + "margin_dpo/margin_std": 487.4764404296875, + "step": 522 + }, + { + "KL/chosen_KL_mean": -603.8565673828125, + "KL/mean": -775.8800659179688, + "KL/rejected_KL_mean": -947.9034423828125, + "KL/std": 541.7845458984375, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0010489480337128043, + "fcm_dpo/delta": 0.040566716343164444, + "fcm_dpo/margin": 344.0469665527344, + "fcm_dpo/q_t": 0.41834262013435364, + "grad_norm": 44.656070709228516, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.9243185520172119, + "logits/rejected": -0.9296277761459351, + "logps/chosen": -665.6261596679688, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1031.664794921875, + "loss": 1.141, + "margin_dpo/margin_mean": 344.0469665527344, + "margin_dpo/margin_std": 571.0057373046875, + "step": 523 + }, + { + "KL/chosen_KL_mean": -593.1018676757812, + "KL/mean": -777.8427734375, + "KL/rejected_KL_mean": -962.583740234375, + "KL/std": 503.5099792480469, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.0010478496551513672, + "fcm_dpo/delta": 0.012942522764205933, + "fcm_dpo/margin": 369.4818115234375, + "fcm_dpo/q_t": 0.4089270234107971, + "grad_norm": 41.08769607543945, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -0.9892967939376831, + "logits/rejected": -0.9853494167327881, + "logps/chosen": -671.1739501953125, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1043.8857421875, + "loss": 1.1077, + "margin_dpo/margin_mean": 369.4818115234375, + "margin_dpo/margin_std": 508.42706298828125, + "step": 524 + }, + { + "KL/chosen_KL_mean": -603.3177490234375, + "KL/mean": -833.9947509765625, + "KL/rejected_KL_mean": -1064.6717529296875, + "KL/std": 530.2716674804688, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.001036688918247819, + "fcm_dpo/delta": -0.08283720910549164, + "fcm_dpo/margin": 461.35406494140625, + "fcm_dpo/q_t": 0.392111599445343, + "grad_norm": 34.44672393798828, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.9192453622817993, + "logits/rejected": -0.9561092257499695, + "logps/chosen": -654.1455688476562, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1164.7247314453125, + "loss": 1.0422, + "margin_dpo/margin_mean": 461.35406494140625, + "margin_dpo/margin_std": 564.3133544921875, + "step": 525 + }, + { + "KL/chosen_KL_mean": -662.8272094726562, + "KL/mean": -885.3590698242188, + "KL/rejected_KL_mean": -1107.8909912109375, + "KL/std": 523.77734375, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0010306935291737318, + "fcm_dpo/delta": -0.06160289794206619, + "fcm_dpo/margin": 445.06378173828125, + "fcm_dpo/q_t": 0.39464327692985535, + "grad_norm": 24.1711483001709, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.940832257270813, + "logits/rejected": -0.937059223651886, + "logps/chosen": -725.9944458007812, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1194.2003173828125, + "loss": 1.0666, + "margin_dpo/margin_mean": 445.06378173828125, + "margin_dpo/margin_std": 587.0912475585938, + "step": 526 + }, + { + "KL/chosen_KL_mean": -692.7408447265625, + "KL/mean": -839.8806762695312, + "KL/rejected_KL_mean": -987.0204467773438, + "KL/std": 548.38232421875, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0010303595336154103, + "fcm_dpo/delta": 0.03291347250342369, + "fcm_dpo/margin": 294.27960205078125, + "fcm_dpo/q_t": 0.42890849709510803, + "grad_norm": 39.53373336791992, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.142526388168335, + "logits/rejected": -1.1143360137939453, + "logps/chosen": -763.8695068359375, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1065.363037109375, + "loss": 1.1612, + "margin_dpo/margin_mean": 294.27960205078125, + "margin_dpo/margin_std": 465.8953552246094, + "step": 527 + }, + { + "KL/chosen_KL_mean": -705.5905151367188, + "KL/mean": -885.827880859375, + "KL/rejected_KL_mean": -1066.065185546875, + "KL/std": 527.7647705078125, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0010336707346141338, + "fcm_dpo/delta": 0.028455456718802452, + "fcm_dpo/margin": 360.4747314453125, + "fcm_dpo/q_t": 0.4175037741661072, + "grad_norm": 55.92026138305664, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -1.0248111486434937, + "logits/rejected": -1.0066437721252441, + "logps/chosen": -773.679931640625, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1159.9752197265625, + "loss": 1.1469, + "margin_dpo/margin_mean": 360.4747009277344, + "margin_dpo/margin_std": 632.50146484375, + "step": 528 + }, + { + "KL/chosen_KL_mean": -829.6602172851562, + "KL/mean": -960.8089599609375, + "KL/rejected_KL_mean": -1091.957763671875, + "KL/std": 490.57574462890625, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.0010418513556942344, + "fcm_dpo/delta": 0.02700192667543888, + "fcm_dpo/margin": 262.2973937988281, + "fcm_dpo/q_t": 0.4371851682662964, + "grad_norm": 40.754058837890625, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.0900201797485352, + "logits/rejected": -1.1030248403549194, + "logps/chosen": -885.2351684570312, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1181.166748046875, + "loss": 1.2292, + "margin_dpo/margin_mean": 262.2973937988281, + "margin_dpo/margin_std": 609.2044677734375, + "step": 529 + }, + { + "KL/chosen_KL_mean": -700.2659912109375, + "KL/mean": -910.9698486328125, + "KL/rejected_KL_mean": -1121.673828125, + "KL/std": 593.5812377929688, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0010373882250860333, + "fcm_dpo/delta": -0.0388498418033123, + "fcm_dpo/margin": 421.4078369140625, + "fcm_dpo/q_t": 0.4044303894042969, + "grad_norm": 43.0464973449707, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.9657949805259705, + "logits/rejected": -0.9637165069580078, + "logps/chosen": -747.8673706054688, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1208.9583740234375, + "loss": 1.1244, + "margin_dpo/margin_mean": 421.4078369140625, + "margin_dpo/margin_std": 736.1807250976562, + "step": 530 + }, + { + "KL/chosen_KL_mean": -798.591796875, + "KL/mean": -973.8945922851562, + "KL/rejected_KL_mean": -1149.197265625, + "KL/std": 625.2605590820312, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0010416710283607244, + "fcm_dpo/delta": 0.035919878631830215, + "fcm_dpo/margin": 350.6055908203125, + "fcm_dpo/q_t": 0.4186422824859619, + "grad_norm": 64.89240264892578, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.0689399242401123, + "logits/rejected": -1.0525445938110352, + "logps/chosen": -853.8378295898438, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1219.80322265625, + "loss": 1.1474, + "margin_dpo/margin_mean": 350.6055908203125, + "margin_dpo/margin_std": 611.898193359375, + "step": 531 + }, + { + "KL/chosen_KL_mean": -790.3135986328125, + "KL/mean": -981.876953125, + "KL/rejected_KL_mean": -1173.4403076171875, + "KL/std": 593.156494140625, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0010262735886499286, + "fcm_dpo/delta": -0.07239500433206558, + "fcm_dpo/margin": 383.12664794921875, + "fcm_dpo/q_t": 0.4123035669326782, + "grad_norm": 89.1617660522461, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.006425142288208, + "logits/rejected": -0.9871149063110352, + "logps/chosen": -860.599609375, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1260.0316162109375, + "loss": 1.158, + "margin_dpo/margin_mean": 383.1266784667969, + "margin_dpo/margin_std": 710.3892211914062, + "step": 532 + }, + { + "KL/chosen_KL_mean": -741.3399047851562, + "KL/mean": -882.526123046875, + "KL/rejected_KL_mean": -1023.7123413085938, + "KL/std": 533.8526000976562, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0010256596142426133, + "fcm_dpo/delta": -0.005985884461551905, + "fcm_dpo/margin": 282.3724365234375, + "fcm_dpo/q_t": 0.43500691652297974, + "grad_norm": 51.34868621826172, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -1.0395231246948242, + "logits/rejected": -1.0229017734527588, + "logps/chosen": -806.1950073242188, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1100.3004150390625, + "loss": 1.2188, + "margin_dpo/margin_mean": 282.3724365234375, + "margin_dpo/margin_std": 638.7913818359375, + "step": 533 + }, + { + "KL/chosen_KL_mean": -757.0450439453125, + "KL/mean": -949.7239379882812, + "KL/rejected_KL_mean": -1142.40283203125, + "KL/std": 579.4583129882812, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.0010111816227436066, + "fcm_dpo/delta": -0.06808765977621078, + "fcm_dpo/margin": 385.357666015625, + "fcm_dpo/q_t": 0.41200900077819824, + "grad_norm": 42.10295104980469, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.0338828563690186, + "logits/rejected": -1.0336880683898926, + "logps/chosen": -817.1644287109375, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1220.9462890625, + "loss": 1.1239, + "margin_dpo/margin_mean": 385.357666015625, + "margin_dpo/margin_std": 604.0343017578125, + "step": 534 + }, + { + "KL/chosen_KL_mean": -655.06689453125, + "KL/mean": -875.2470703125, + "KL/rejected_KL_mean": -1095.42724609375, + "KL/std": 564.0452270507812, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0010023104259744287, + "fcm_dpo/delta": -0.08850973099470139, + "fcm_dpo/margin": 440.36041259765625, + "fcm_dpo/q_t": 0.40085458755493164, + "grad_norm": 27.047229766845703, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.9276406764984131, + "logits/rejected": -0.9336996078491211, + "logps/chosen": -709.3970947265625, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1191.73486328125, + "loss": 1.0787, + "margin_dpo/margin_mean": 440.36041259765625, + "margin_dpo/margin_std": 595.4772338867188, + "step": 535 + }, + { + "KL/chosen_KL_mean": -578.295166015625, + "KL/mean": -842.19482421875, + "KL/rejected_KL_mean": -1106.0946044921875, + "KL/std": 630.71435546875, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.000976434675976634, + "fcm_dpo/delta": -0.12154753506183624, + "fcm_dpo/margin": 527.7994384765625, + "fcm_dpo/q_t": 0.3903592824935913, + "grad_norm": 36.46469497680664, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.8955657482147217, + "logits/rejected": -0.9310792684555054, + "logps/chosen": -625.375732421875, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1195.1925048828125, + "loss": 1.0477, + "margin_dpo/margin_mean": 527.7994384765625, + "margin_dpo/margin_std": 763.1939697265625, + "step": 536 + }, + { + "KL/chosen_KL_mean": -604.92919921875, + "KL/mean": -777.3243408203125, + "KL/rejected_KL_mean": -949.7193603515625, + "KL/std": 505.7130432128906, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.000967323430813849, + "fcm_dpo/delta": -0.023331381380558014, + "fcm_dpo/margin": 344.7901916503906, + "fcm_dpo/q_t": 0.42241036891937256, + "grad_norm": 58.42335891723633, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.9586353898048401, + "logits/rejected": -0.9458541870117188, + "logps/chosen": -662.6766357421875, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1020.1577758789062, + "loss": 1.1678, + "margin_dpo/margin_mean": 344.7901916503906, + "margin_dpo/margin_std": 616.85205078125, + "step": 537 + }, + { + "KL/chosen_KL_mean": -759.1534423828125, + "KL/mean": -956.6287841796875, + "KL/rejected_KL_mean": -1154.10400390625, + "KL/std": 567.6751708984375, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0009653990855440497, + "fcm_dpo/delta": 0.01937374286353588, + "fcm_dpo/margin": 394.95062255859375, + "fcm_dpo/q_t": 0.4134565591812134, + "grad_norm": 34.96083068847656, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.9518255591392517, + "logits/rejected": -0.9429917335510254, + "logps/chosen": -825.5693359375, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1238.3321533203125, + "loss": 1.1168, + "margin_dpo/margin_mean": 394.95062255859375, + "margin_dpo/margin_std": 592.9100341796875, + "step": 538 + }, + { + "KL/chosen_KL_mean": -631.4656982421875, + "KL/mean": -819.9053955078125, + "KL/rejected_KL_mean": -1008.3450317382812, + "KL/std": 449.70758056640625, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.0009752546902745962, + "fcm_dpo/delta": 0.033576734364032745, + "fcm_dpo/margin": 376.8792724609375, + "fcm_dpo/q_t": 0.41553953289985657, + "grad_norm": 41.08725357055664, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.9777531027793884, + "logits/rejected": -0.9721415638923645, + "logps/chosen": -689.9586181640625, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1100.198974609375, + "loss": 1.1109, + "margin_dpo/margin_mean": 376.8793029785156, + "margin_dpo/margin_std": 513.1068115234375, + "step": 539 + }, + { + "KL/chosen_KL_mean": -618.1189575195312, + "KL/mean": -870.6326904296875, + "KL/rejected_KL_mean": -1123.1463623046875, + "KL/std": 565.5108642578125, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0009656627662479877, + "fcm_dpo/delta": -0.09212129563093185, + "fcm_dpo/margin": 505.02740478515625, + "fcm_dpo/q_t": 0.39293187856674194, + "grad_norm": 45.533443450927734, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.8891763687133789, + "logits/rejected": -0.9183998107910156, + "logps/chosen": -681.6015014648438, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1239.576416015625, + "loss": 1.0549, + "margin_dpo/margin_mean": 505.02740478515625, + "margin_dpo/margin_std": 674.9073486328125, + "step": 540 + }, + { + "KL/chosen_KL_mean": -755.3883666992188, + "KL/mean": -877.87890625, + "KL/rejected_KL_mean": -1000.369384765625, + "KL/std": 510.2044982910156, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.0009612381691113114, + "fcm_dpo/delta": 0.019296646118164062, + "fcm_dpo/margin": 244.98106384277344, + "fcm_dpo/q_t": 0.4461899995803833, + "grad_norm": 73.64654541015625, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.0284849405288696, + "logits/rejected": -1.0217002630233765, + "logps/chosen": -833.668701171875, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1103.64306640625, + "loss": 1.2495, + "margin_dpo/margin_mean": 244.98106384277344, + "margin_dpo/margin_std": 610.431884765625, + "step": 541 + }, + { + "KL/chosen_KL_mean": -620.0665283203125, + "KL/mean": -864.9902954101562, + "KL/rejected_KL_mean": -1109.9140625, + "KL/std": 563.8133544921875, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.0009439120767638087, + "fcm_dpo/delta": -0.06731873750686646, + "fcm_dpo/margin": 489.8475341796875, + "fcm_dpo/q_t": 0.39619243144989014, + "grad_norm": 58.94521713256836, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -0.9920356869697571, + "logits/rejected": -1.0281641483306885, + "logps/chosen": -677.551513671875, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1206.38916015625, + "loss": 1.0718, + "margin_dpo/margin_mean": 489.8475341796875, + "margin_dpo/margin_std": 671.4158325195312, + "step": 542 + }, + { + "KL/chosen_KL_mean": -713.327392578125, + "KL/mean": -891.6700439453125, + "KL/rejected_KL_mean": -1070.0128173828125, + "KL/std": 666.244140625, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0009496011771261692, + "fcm_dpo/delta": 0.006418236065655947, + "fcm_dpo/margin": 356.6854553222656, + "fcm_dpo/q_t": 0.431581050157547, + "grad_norm": 36.109249114990234, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.9449714422225952, + "logits/rejected": -0.9770427942276001, + "logps/chosen": -773.94482421875, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1175.61181640625, + "loss": 1.1939, + "margin_dpo/margin_mean": 356.68548583984375, + "margin_dpo/margin_std": 781.5635986328125, + "step": 543 + }, + { + "KL/chosen_KL_mean": -709.9495849609375, + "KL/mean": -913.177001953125, + "KL/rejected_KL_mean": -1116.4044189453125, + "KL/std": 544.1904296875, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.000950878718867898, + "fcm_dpo/delta": 0.014052917249500751, + "fcm_dpo/margin": 406.45489501953125, + "fcm_dpo/q_t": 0.4121043086051941, + "grad_norm": 32.15199279785156, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.9631912708282471, + "logits/rejected": -0.9874793887138367, + "logps/chosen": -769.5919189453125, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1217.359130859375, + "loss": 1.117, + "margin_dpo/margin_mean": 406.4548645019531, + "margin_dpo/margin_std": 620.988037109375, + "step": 544 + }, + { + "KL/chosen_KL_mean": -656.1561279296875, + "KL/mean": -894.4105834960938, + "KL/rejected_KL_mean": -1132.6650390625, + "KL/std": 548.3407592773438, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0009447300108149648, + "fcm_dpo/delta": -0.052544862031936646, + "fcm_dpo/margin": 476.5089111328125, + "fcm_dpo/q_t": 0.39853060245513916, + "grad_norm": 32.023895263671875, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.878632664680481, + "logits/rejected": -0.9118402004241943, + "logps/chosen": -723.8047485351562, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1228.572998046875, + "loss": 1.0661, + "margin_dpo/margin_mean": 476.5089111328125, + "margin_dpo/margin_std": 627.6030883789062, + "step": 545 + }, + { + "KL/chosen_KL_mean": -634.7700805664062, + "KL/mean": -784.2230834960938, + "KL/rejected_KL_mean": -933.6761474609375, + "KL/std": 473.97064208984375, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.0009423188748769462, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 298.906005859375, + "fcm_dpo/q_t": 0.4325985908508301, + "grad_norm": 33.06241226196289, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.9311190843582153, + "logits/rejected": -0.9174859523773193, + "logps/chosen": -685.5142822265625, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1015.5423583984375, + "loss": 1.1822, + "margin_dpo/margin_mean": 298.9060363769531, + "margin_dpo/margin_std": 518.330322265625, + "step": 546 + }, + { + "KL/chosen_KL_mean": -620.3345947265625, + "KL/mean": -819.4880981445312, + "KL/rejected_KL_mean": -1018.6416015625, + "KL/std": 517.9036865234375, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.0009470967925153673, + "fcm_dpo/delta": 0.023572035133838654, + "fcm_dpo/margin": 398.30706787109375, + "fcm_dpo/q_t": 0.41405177116394043, + "grad_norm": 47.677310943603516, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.8830410242080688, + "logits/rejected": -0.8951892852783203, + "logps/chosen": -694.0222778320312, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1109.403076171875, + "loss": 1.117, + "margin_dpo/margin_mean": 398.3070983886719, + "margin_dpo/margin_std": 589.7435302734375, + "step": 547 + }, + { + "KL/chosen_KL_mean": -636.1212158203125, + "KL/mean": -824.2808837890625, + "KL/rejected_KL_mean": -1012.4405517578125, + "KL/std": 554.9188232421875, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0009525719797238708, + "fcm_dpo/delta": 0.04308447614312172, + "fcm_dpo/margin": 376.3193359375, + "fcm_dpo/q_t": 0.4210885763168335, + "grad_norm": 27.067203521728516, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -0.9821116924285889, + "logits/rejected": -0.9854071736335754, + "logps/chosen": -701.3675537109375, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1106.55859375, + "loss": 1.1318, + "margin_dpo/margin_mean": 376.3193359375, + "margin_dpo/margin_std": 604.8456420898438, + "step": 548 + }, + { + "KL/chosen_KL_mean": -677.4143676757812, + "KL/mean": -818.0396728515625, + "KL/rejected_KL_mean": -958.6650390625, + "KL/std": 462.95379638671875, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0009576628217473626, + "fcm_dpo/delta": 0.028166765347123146, + "fcm_dpo/margin": 281.2506103515625, + "fcm_dpo/q_t": 0.4384850859642029, + "grad_norm": 56.85281753540039, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.988927960395813, + "logits/rejected": -0.9804089665412903, + "logps/chosen": -726.626708984375, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1032.5753173828125, + "loss": 1.2049, + "margin_dpo/margin_mean": 281.2506103515625, + "margin_dpo/margin_std": 560.0017700195312, + "step": 549 + }, + { + "KL/chosen_KL_mean": -667.2991943359375, + "KL/mean": -834.825439453125, + "KL/rejected_KL_mean": -1002.3516845703125, + "KL/std": 502.79473876953125, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.0009710068115964532, + "fcm_dpo/delta": 0.07720610499382019, + "fcm_dpo/margin": 335.052490234375, + "fcm_dpo/q_t": 0.42524394392967224, + "grad_norm": 38.28554916381836, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.9163668155670166, + "logits/rejected": -0.9437706470489502, + "logps/chosen": -724.1061401367188, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1097.4775390625, + "loss": 1.1504, + "margin_dpo/margin_mean": 335.052490234375, + "margin_dpo/margin_std": 539.7377319335938, + "step": 550 + }, + { + "KL/chosen_KL_mean": -577.0341796875, + "KL/mean": -854.4708251953125, + "KL/rejected_KL_mean": -1131.907470703125, + "KL/std": 559.3414306640625, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0009543923079036176, + "fcm_dpo/delta": -0.13702905178070068, + "fcm_dpo/margin": 554.873291015625, + "fcm_dpo/q_t": 0.3782382309436798, + "grad_norm": 79.90360260009766, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.8979976177215576, + "logits/rejected": -0.9577436447143555, + "logps/chosen": -636.1405029296875, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1243.580322265625, + "loss": 0.9909, + "margin_dpo/margin_mean": 554.873291015625, + "margin_dpo/margin_std": 563.7725830078125, + "step": 551 + }, + { + "KL/chosen_KL_mean": -562.810791015625, + "KL/mean": -867.3089599609375, + "KL/rejected_KL_mean": -1171.80712890625, + "KL/std": 634.3409423828125, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.0009175877785310149, + "fcm_dpo/delta": -0.1709892302751541, + "fcm_dpo/margin": 608.996337890625, + "fcm_dpo/q_t": 0.377045214176178, + "grad_norm": 59.48480987548828, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.9460190534591675, + "logits/rejected": -0.991344153881073, + "logps/chosen": -625.1654052734375, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1276.369140625, + "loss": 1.0008, + "margin_dpo/margin_mean": 608.9962768554688, + "margin_dpo/margin_std": 723.9281005859375, + "step": 552 + }, + { + "KL/chosen_KL_mean": -642.0367431640625, + "KL/mean": -801.6822509765625, + "KL/rejected_KL_mean": -961.3277587890625, + "KL/std": 524.64990234375, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0009227884584106505, + "fcm_dpo/delta": 0.03126968815922737, + "fcm_dpo/margin": 319.291015625, + "fcm_dpo/q_t": 0.43378591537475586, + "grad_norm": 30.559711456298828, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.9423331618309021, + "logits/rejected": -0.9393686652183533, + "logps/chosen": -710.2955932617188, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1059.4249267578125, + "loss": 1.1781, + "margin_dpo/margin_mean": 319.291015625, + "margin_dpo/margin_std": 576.5843505859375, + "step": 553 + }, + { + "KL/chosen_KL_mean": -678.570068359375, + "KL/mean": -886.33837890625, + "KL/rejected_KL_mean": -1094.106689453125, + "KL/std": 605.6937866210938, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0009171630954369903, + "fcm_dpo/delta": -0.06133558973670006, + "fcm_dpo/margin": 415.53662109375, + "fcm_dpo/q_t": 0.4128907322883606, + "grad_norm": 140.4740753173828, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.9348787069320679, + "logits/rejected": -0.914147138595581, + "logps/chosen": -746.5177612304688, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1183.889404296875, + "loss": 1.1642, + "margin_dpo/margin_mean": 415.53662109375, + "margin_dpo/margin_std": 789.6375732421875, + "step": 554 + }, + { + "KL/chosen_KL_mean": -660.2467041015625, + "KL/mean": -918.8154296875, + "KL/rejected_KL_mean": -1177.38427734375, + "KL/std": 600.5507202148438, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.000903658801689744, + "fcm_dpo/delta": -0.07055527716875076, + "fcm_dpo/margin": 517.1375732421875, + "fcm_dpo/q_t": 0.39831459522247314, + "grad_norm": 50.94032669067383, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -0.9928549528121948, + "logits/rejected": -1.0527600049972534, + "logps/chosen": -713.5771484375, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1285.863525390625, + "loss": 1.0801, + "margin_dpo/margin_mean": 517.1375122070312, + "margin_dpo/margin_std": 772.34375, + "step": 555 + }, + { + "KL/chosen_KL_mean": -650.8236083984375, + "KL/mean": -807.4058837890625, + "KL/rejected_KL_mean": -963.9881591796875, + "KL/std": 467.30462646484375, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.0008987652254290879, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 313.16455078125, + "fcm_dpo/q_t": 0.4336881637573242, + "grad_norm": 32.71649169921875, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -0.9493398666381836, + "logits/rejected": -0.9457892179489136, + "logps/chosen": -709.4681396484375, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1065.32861328125, + "loss": 1.1679, + "margin_dpo/margin_mean": 313.1645812988281, + "margin_dpo/margin_std": 474.4263916015625, + "step": 556 + }, + { + "KL/chosen_KL_mean": -724.3292236328125, + "KL/mean": -955.7681884765625, + "KL/rejected_KL_mean": -1187.207275390625, + "KL/std": 648.9892578125, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.0008952942444011569, + "fcm_dpo/delta": -0.015113058499991894, + "fcm_dpo/margin": 462.87799072265625, + "fcm_dpo/q_t": 0.4079136848449707, + "grad_norm": 66.14971160888672, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -0.9885218143463135, + "logits/rejected": -1.0276739597320557, + "logps/chosen": -792.169921875, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1297.1468505859375, + "loss": 1.1132, + "margin_dpo/margin_mean": 462.87799072265625, + "margin_dpo/margin_std": 731.037353515625, + "step": 557 + }, + { + "KL/chosen_KL_mean": -647.0894775390625, + "KL/mean": -956.5216064453125, + "KL/rejected_KL_mean": -1265.95361328125, + "KL/std": 593.8663330078125, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0008764144731685519, + "fcm_dpo/delta": -0.15052077174186707, + "fcm_dpo/margin": 618.8641967773438, + "fcm_dpo/q_t": 0.37401190400123596, + "grad_norm": 33.20355987548828, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.054278016090393, + "logits/rejected": -1.0769346952438354, + "logps/chosen": -709.457763671875, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1368.11474609375, + "loss": 0.9959, + "margin_dpo/margin_mean": 618.8641967773438, + "margin_dpo/margin_std": 669.4278564453125, + "step": 558 + }, + { + "KL/chosen_KL_mean": -734.8444213867188, + "KL/mean": -988.641357421875, + "KL/rejected_KL_mean": -1242.4383544921875, + "KL/std": 614.178466796875, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0008658560691401362, + "fcm_dpo/delta": -0.04131707921624184, + "fcm_dpo/margin": 507.5938720703125, + "fcm_dpo/q_t": 0.4007589817047119, + "grad_norm": 33.70786666870117, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.1651673316955566, + "logits/rejected": -1.172461748123169, + "logps/chosen": -795.5968017578125, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1335.880615234375, + "loss": 1.0723, + "margin_dpo/margin_mean": 507.5938720703125, + "margin_dpo/margin_std": 675.1575317382812, + "step": 559 + }, + { + "KL/chosen_KL_mean": -681.6367797851562, + "KL/mean": -864.1337890625, + "KL/rejected_KL_mean": -1046.630859375, + "KL/std": 531.2283935546875, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0008626263006590307, + "fcm_dpo/delta": 0.0017498014494776726, + "fcm_dpo/margin": 364.99407958984375, + "fcm_dpo/q_t": 0.4270641505718231, + "grad_norm": 41.76873016357422, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.8998410105705261, + "logits/rejected": -0.8779373168945312, + "logps/chosen": -739.7406005859375, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1126.6220703125, + "loss": 1.157, + "margin_dpo/margin_mean": 364.99407958984375, + "margin_dpo/margin_std": 581.9285888671875, + "step": 560 + }, + { + "KL/chosen_KL_mean": -781.733642578125, + "KL/mean": -942.5443115234375, + "KL/rejected_KL_mean": -1103.3548583984375, + "KL/std": 551.874755859375, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0008657281287014484, + "fcm_dpo/delta": 0.03582902252674103, + "fcm_dpo/margin": 321.62115478515625, + "fcm_dpo/q_t": 0.4346877336502075, + "grad_norm": 38.080963134765625, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.076653242111206, + "logits/rejected": -1.0977752208709717, + "logps/chosen": -832.6466064453125, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1202.4234619140625, + "loss": 1.2107, + "margin_dpo/margin_mean": 321.6212158203125, + "margin_dpo/margin_std": 692.325439453125, + "step": 561 + }, + { + "KL/chosen_KL_mean": -742.1090087890625, + "KL/mean": -963.9869384765625, + "KL/rejected_KL_mean": -1185.86474609375, + "KL/std": 610.6798095703125, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0008610176737420261, + "fcm_dpo/delta": -0.045161668211221695, + "fcm_dpo/margin": 443.755859375, + "fcm_dpo/q_t": 0.41347572207450867, + "grad_norm": 34.880821228027344, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.0064786672592163, + "logits/rejected": -1.0152138471603394, + "logps/chosen": -801.5734252929688, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1282.407470703125, + "loss": 1.1158, + "margin_dpo/margin_mean": 443.755859375, + "margin_dpo/margin_std": 665.2860107421875, + "step": 562 + }, + { + "KL/chosen_KL_mean": -824.5828857421875, + "KL/mean": -985.7767333984375, + "KL/rejected_KL_mean": -1146.9705810546875, + "KL/std": 655.7936401367188, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0008587371557950974, + "fcm_dpo/delta": -0.026556432247161865, + "fcm_dpo/margin": 322.38763427734375, + "fcm_dpo/q_t": 0.43178072571754456, + "grad_norm": 40.33382797241211, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.031575322151184, + "logits/rejected": -1.0388014316558838, + "logps/chosen": -885.1911010742188, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1241.538330078125, + "loss": 1.2375, + "margin_dpo/margin_mean": 322.3876647949219, + "margin_dpo/margin_std": 762.0474853515625, + "step": 563 + }, + { + "KL/chosen_KL_mean": -737.0413208007812, + "KL/mean": -948.6756591796875, + "KL/rejected_KL_mean": -1160.31005859375, + "KL/std": 580.0350341796875, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0008632763056084514, + "fcm_dpo/delta": 0.035682305693626404, + "fcm_dpo/margin": 423.2686767578125, + "fcm_dpo/q_t": 0.4167470932006836, + "grad_norm": 41.73084259033203, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.1041316986083984, + "logits/rejected": -1.0973750352859497, + "logps/chosen": -811.3096923828125, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1253.548095703125, + "loss": 1.1144, + "margin_dpo/margin_mean": 423.2686767578125, + "margin_dpo/margin_std": 578.4511108398438, + "step": 564 + }, + { + "KL/chosen_KL_mean": -764.9254760742188, + "KL/mean": -982.154052734375, + "KL/rejected_KL_mean": -1199.382568359375, + "KL/std": 665.6796875, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.0008658579317852855, + "fcm_dpo/delta": 0.0247506033629179, + "fcm_dpo/margin": 434.45709228515625, + "fcm_dpo/q_t": 0.4183410406112671, + "grad_norm": 58.21854782104492, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.0195878744125366, + "logits/rejected": -1.0214147567749023, + "logps/chosen": -833.9454345703125, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1285.16162109375, + "loss": 1.1545, + "margin_dpo/margin_mean": 434.45709228515625, + "margin_dpo/margin_std": 804.218994140625, + "step": 565 + }, + { + "KL/chosen_KL_mean": -745.803466796875, + "KL/mean": -994.66552734375, + "KL/rejected_KL_mean": -1243.527587890625, + "KL/std": 679.287109375, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0008602312882430851, + "fcm_dpo/delta": -0.0771884024143219, + "fcm_dpo/margin": 497.72418212890625, + "fcm_dpo/q_t": 0.4038347601890564, + "grad_norm": 32.46363067626953, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.0346779823303223, + "logits/rejected": -1.0605592727661133, + "logps/chosen": -812.3487548828125, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1347.39697265625, + "loss": 1.1102, + "margin_dpo/margin_mean": 497.72418212890625, + "margin_dpo/margin_std": 776.3060302734375, + "step": 566 + }, + { + "KL/chosen_KL_mean": -705.974853515625, + "KL/mean": -888.8309936523438, + "KL/rejected_KL_mean": -1071.6871337890625, + "KL/std": 467.6333312988281, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0008642717730253935, + "fcm_dpo/delta": 0.08670350164175034, + "fcm_dpo/margin": 365.71234130859375, + "fcm_dpo/q_t": 0.42557185888290405, + "grad_norm": 27.69707489013672, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.1318271160125732, + "logits/rejected": -1.1388347148895264, + "logps/chosen": -758.8331298828125, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1157.05810546875, + "loss": 1.1425, + "margin_dpo/margin_mean": 365.71234130859375, + "margin_dpo/margin_std": 517.5740966796875, + "step": 567 + }, + { + "KL/chosen_KL_mean": -687.024169921875, + "KL/mean": -942.22265625, + "KL/rejected_KL_mean": -1197.421142578125, + "KL/std": 547.3006591796875, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0008622035384178162, + "fcm_dpo/delta": -0.041977256536483765, + "fcm_dpo/margin": 510.39691162109375, + "fcm_dpo/q_t": 0.39740312099456787, + "grad_norm": 30.58122444152832, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.017067790031433, + "logits/rejected": -1.0572055578231812, + "logps/chosen": -732.216552734375, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1286.513427734375, + "loss": 1.0517, + "margin_dpo/margin_mean": 510.39691162109375, + "margin_dpo/margin_std": 579.4642333984375, + "step": 568 + }, + { + "KL/chosen_KL_mean": -775.1475830078125, + "KL/mean": -990.3033447265625, + "KL/rejected_KL_mean": -1205.4591064453125, + "KL/std": 708.2174072265625, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.0008574404055252671, + "fcm_dpo/delta": -0.044561613351106644, + "fcm_dpo/margin": 430.3114013671875, + "fcm_dpo/q_t": 0.41521763801574707, + "grad_norm": 47.21596908569336, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.0413999557495117, + "logits/rejected": -1.0494320392608643, + "logps/chosen": -838.8681640625, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1284.562255859375, + "loss": 1.1468, + "margin_dpo/margin_mean": 430.3114013671875, + "margin_dpo/margin_std": 730.2767944335938, + "step": 569 + }, + { + "KL/chosen_KL_mean": -728.922119140625, + "KL/mean": -917.3353271484375, + "KL/rejected_KL_mean": -1105.7484130859375, + "KL/std": 558.15673828125, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0008628984214738011, + "fcm_dpo/delta": 0.07739873230457306, + "fcm_dpo/margin": 376.82635498046875, + "fcm_dpo/q_t": 0.42575061321258545, + "grad_norm": 37.80527114868164, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.1605284214019775, + "logits/rejected": -1.1561046838760376, + "logps/chosen": -790.5367431640625, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1187.890380859375, + "loss": 1.1521, + "margin_dpo/margin_mean": 376.82635498046875, + "margin_dpo/margin_std": 599.8521728515625, + "step": 570 + }, + { + "KL/chosen_KL_mean": -738.0014038085938, + "KL/mean": -964.416259765625, + "KL/rejected_KL_mean": -1190.8310546875, + "KL/std": 575.76953125, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0008670316310599446, + "fcm_dpo/delta": 0.007670203689485788, + "fcm_dpo/margin": 452.8296203613281, + "fcm_dpo/q_t": 0.4113842844963074, + "grad_norm": 41.54902267456055, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.000368356704712, + "logits/rejected": -1.0172909498214722, + "logps/chosen": -791.0554809570312, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1282.16796875, + "loss": 1.1223, + "margin_dpo/margin_mean": 452.82958984375, + "margin_dpo/margin_std": 722.2899169921875, + "step": 571 + }, + { + "KL/chosen_KL_mean": -787.4087524414062, + "KL/mean": -1028.535888671875, + "KL/rejected_KL_mean": -1269.6630859375, + "KL/std": 676.5159912109375, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0008637818391434848, + "fcm_dpo/delta": -0.01746346987783909, + "fcm_dpo/margin": 482.25433349609375, + "fcm_dpo/q_t": 0.4096788465976715, + "grad_norm": 29.09326171875, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.047022819519043, + "logits/rejected": -1.080221176147461, + "logps/chosen": -835.8680419921875, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1353.22021484375, + "loss": 1.1235, + "margin_dpo/margin_mean": 482.25433349609375, + "margin_dpo/margin_std": 820.7142333984375, + "step": 572 + }, + { + "KL/chosen_KL_mean": -761.902587890625, + "KL/mean": -973.8734741210938, + "KL/rejected_KL_mean": -1185.844482421875, + "KL/std": 597.7796630859375, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0008704299107193947, + "fcm_dpo/delta": 0.03210826218128204, + "fcm_dpo/margin": 423.94183349609375, + "fcm_dpo/q_t": 0.41670119762420654, + "grad_norm": 28.381221771240234, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.0847845077514648, + "logits/rejected": -1.0967497825622559, + "logps/chosen": -813.5252075195312, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1271.16943359375, + "loss": 1.1126, + "margin_dpo/margin_mean": 423.94183349609375, + "margin_dpo/margin_std": 602.698974609375, + "step": 573 + }, + { + "KL/chosen_KL_mean": -695.9632568359375, + "KL/mean": -926.2237548828125, + "KL/rejected_KL_mean": -1156.4842529296875, + "KL/std": 547.908935546875, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0008723270148038864, + "fcm_dpo/delta": -0.0018760263919830322, + "fcm_dpo/margin": 460.52093505859375, + "fcm_dpo/q_t": 0.4079146385192871, + "grad_norm": 37.186824798583984, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.0834007263183594, + "logits/rejected": -1.1153415441513062, + "logps/chosen": -747.0077514648438, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1249.2906494140625, + "loss": 1.0967, + "margin_dpo/margin_mean": 460.52093505859375, + "margin_dpo/margin_std": 635.26416015625, + "step": 574 + }, + { + "KL/chosen_KL_mean": -751.2843017578125, + "KL/mean": -964.8988037109375, + "KL/rejected_KL_mean": -1178.51318359375, + "KL/std": 560.852783203125, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0008594076498411596, + "fcm_dpo/delta": -0.0640491247177124, + "fcm_dpo/margin": 427.22900390625, + "fcm_dpo/q_t": 0.4158279597759247, + "grad_norm": 36.509559631347656, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -0.9182157516479492, + "logits/rejected": -0.9273508191108704, + "logps/chosen": -823.074462890625, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1273.8994140625, + "loss": 1.1457, + "margin_dpo/margin_mean": 427.22900390625, + "margin_dpo/margin_std": 716.449462890625, + "step": 575 + }, + { + "KL/chosen_KL_mean": -725.522705078125, + "KL/mean": -883.64501953125, + "KL/rejected_KL_mean": -1041.767333984375, + "KL/std": 513.2473754882812, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0008647697395645082, + "fcm_dpo/delta": 0.031099434942007065, + "fcm_dpo/margin": 316.24462890625, + "fcm_dpo/q_t": 0.4367942810058594, + "grad_norm": 39.34665298461914, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.0339372158050537, + "logits/rejected": -1.0641186237335205, + "logps/chosen": -779.7857055664062, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1142.521728515625, + "loss": 1.1957, + "margin_dpo/margin_mean": 316.24462890625, + "margin_dpo/margin_std": 594.4417724609375, + "step": 576 + }, + { + "KL/chosen_KL_mean": -640.4210205078125, + "KL/mean": -851.8929443359375, + "KL/rejected_KL_mean": -1063.3648681640625, + "KL/std": 566.5362548828125, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0008682446205057204, + "fcm_dpo/delta": 0.0340440608561039, + "fcm_dpo/margin": 422.94384765625, + "fcm_dpo/q_t": 0.41654685139656067, + "grad_norm": 31.543575286865234, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -0.9947143793106079, + "logits/rejected": -0.9834187030792236, + "logps/chosen": -702.3306884765625, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1147.435546875, + "loss": 1.129, + "margin_dpo/margin_mean": 422.9438781738281, + "margin_dpo/margin_std": 624.9060668945312, + "step": 577 + }, + { + "KL/chosen_KL_mean": -641.32763671875, + "KL/mean": -868.8936157226562, + "KL/rejected_KL_mean": -1096.45947265625, + "KL/std": 558.311279296875, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.000867625349201262, + "fcm_dpo/delta": 0.004976443946361542, + "fcm_dpo/margin": 455.1319580078125, + "fcm_dpo/q_t": 0.40860503911972046, + "grad_norm": 48.17753982543945, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.9418168663978577, + "logits/rejected": -0.9763994812965393, + "logps/chosen": -690.59130859375, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1179.895751953125, + "loss": 1.0881, + "margin_dpo/margin_mean": 455.1319274902344, + "margin_dpo/margin_std": 565.1869506835938, + "step": 578 + }, + { + "KL/chosen_KL_mean": -744.5780029296875, + "KL/mean": -881.455322265625, + "KL/rejected_KL_mean": -1018.3325805664062, + "KL/std": 579.7168579101562, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0008715449366718531, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 273.7546081542969, + "fcm_dpo/q_t": 0.4439253807067871, + "grad_norm": 56.24394607543945, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -0.9115462899208069, + "logits/rejected": -0.9149183034896851, + "logps/chosen": -803.4638671875, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1113.1202392578125, + "loss": 1.2359, + "margin_dpo/margin_mean": 273.7546081542969, + "margin_dpo/margin_std": 637.7313842773438, + "step": 579 + }, + { + "KL/chosen_KL_mean": -523.5972900390625, + "KL/mean": -783.7306518554688, + "KL/rejected_KL_mean": -1043.864013671875, + "KL/std": 631.0306396484375, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0008624076726846397, + "fcm_dpo/delta": -0.05129069462418556, + "fcm_dpo/margin": 520.266845703125, + "fcm_dpo/q_t": 0.4011837840080261, + "grad_norm": 40.11989212036133, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8757172226905823, + "logits/rejected": -0.9127141237258911, + "logps/chosen": -572.3040771484375, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1125.62255859375, + "loss": 1.0716, + "margin_dpo/margin_mean": 520.266845703125, + "margin_dpo/margin_std": 726.966064453125, + "step": 580 + }, + { + "KL/chosen_KL_mean": -752.8870849609375, + "KL/mean": -931.0899658203125, + "KL/rejected_KL_mean": -1109.292724609375, + "KL/std": 598.118408203125, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.000866269285324961, + "fcm_dpo/delta": 0.04177863150835037, + "fcm_dpo/margin": 356.40576171875, + "fcm_dpo/q_t": 0.43042054772377014, + "grad_norm": 31.875896453857422, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.0664353370666504, + "logits/rejected": -1.078697919845581, + "logps/chosen": -815.109375, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1201.028564453125, + "loss": 1.1768, + "margin_dpo/margin_mean": 356.40576171875, + "margin_dpo/margin_std": 663.83203125, + "step": 581 + }, + { + "KL/chosen_KL_mean": -658.3013916015625, + "KL/mean": -860.1112060546875, + "KL/rejected_KL_mean": -1061.921142578125, + "KL/std": 514.4174194335938, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0008647682843729854, + "fcm_dpo/delta": -0.05920872092247009, + "fcm_dpo/margin": 403.61968994140625, + "fcm_dpo/q_t": 0.41829702258110046, + "grad_norm": 41.649532318115234, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -0.9841645956039429, + "logits/rejected": -1.0088688135147095, + "logps/chosen": -716.530029296875, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1171.99072265625, + "loss": 1.1323, + "margin_dpo/margin_mean": 403.61968994140625, + "margin_dpo/margin_std": 571.9515991210938, + "step": 582 + }, + { + "KL/chosen_KL_mean": -740.55126953125, + "KL/mean": -870.7672729492188, + "KL/rejected_KL_mean": -1000.9832763671875, + "KL/std": 570.4888305664062, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0008597064297646284, + "fcm_dpo/delta": 0.00033876491943374276, + "fcm_dpo/margin": 260.43194580078125, + "fcm_dpo/q_t": 0.4491223990917206, + "grad_norm": 55.23774719238281, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.9683166742324829, + "logits/rejected": -0.9655100107192993, + "logps/chosen": -797.4141235351562, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1089.38720703125, + "loss": 1.2669, + "margin_dpo/margin_mean": 260.43194580078125, + "margin_dpo/margin_std": 698.406005859375, + "step": 583 + }, + { + "KL/chosen_KL_mean": -646.0406494140625, + "KL/mean": -907.0245971679688, + "KL/rejected_KL_mean": -1168.0084228515625, + "KL/std": 585.6754760742188, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.0008540721610188484, + "fcm_dpo/delta": -0.04791495203971863, + "fcm_dpo/margin": 521.9678955078125, + "fcm_dpo/q_t": 0.3981691002845764, + "grad_norm": 31.096391677856445, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -0.9887485504150391, + "logits/rejected": -1.0090022087097168, + "logps/chosen": -702.9413452148438, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1265.64453125, + "loss": 1.0545, + "margin_dpo/margin_mean": 521.9678955078125, + "margin_dpo/margin_std": 617.473876953125, + "step": 584 + }, + { + "KL/chosen_KL_mean": -744.862548828125, + "KL/mean": -982.033203125, + "KL/rejected_KL_mean": -1219.203857421875, + "KL/std": 655.14208984375, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0008483690326102078, + "fcm_dpo/delta": -0.0027475692331790924, + "fcm_dpo/margin": 474.34130859375, + "fcm_dpo/q_t": 0.40947556495666504, + "grad_norm": 44.54747772216797, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.0387095212936401, + "logits/rejected": -1.0658140182495117, + "logps/chosen": -805.8365478515625, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1303.373291015625, + "loss": 1.1254, + "margin_dpo/margin_mean": 474.34124755859375, + "margin_dpo/margin_std": 785.4075927734375, + "step": 585 + }, + { + "KL/chosen_KL_mean": -795.0472412109375, + "KL/mean": -954.4124755859375, + "KL/rejected_KL_mean": -1113.777587890625, + "KL/std": 587.038330078125, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.000851039367262274, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 318.73046875, + "fcm_dpo/q_t": 0.437466561794281, + "grad_norm": 34.602291107177734, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.0131325721740723, + "logits/rejected": -1.0368674993515015, + "logps/chosen": -851.90283203125, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1205.580322265625, + "loss": 1.2037, + "margin_dpo/margin_mean": 318.73046875, + "margin_dpo/margin_std": 639.5387573242188, + "step": 586 + }, + { + "KL/chosen_KL_mean": -549.74853515625, + "KL/mean": -767.3350830078125, + "KL/rejected_KL_mean": -984.921630859375, + "KL/std": 595.6466064453125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.000856140919495374, + "fcm_dpo/delta": 0.028380293399095535, + "fcm_dpo/margin": 435.173095703125, + "fcm_dpo/q_t": 0.41385918855667114, + "grad_norm": 39.022132873535156, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.0584665536880493, + "logits/rejected": -1.0889898538589478, + "logps/chosen": -594.4401245117188, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1067.54541015625, + "loss": 1.1093, + "margin_dpo/margin_mean": 435.1730651855469, + "margin_dpo/margin_std": 596.4622802734375, + "step": 587 + }, + { + "KL/chosen_KL_mean": -722.34521484375, + "KL/mean": -903.4992065429688, + "KL/rejected_KL_mean": -1084.6531982421875, + "KL/std": 518.5870971679688, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0008574275998398662, + "fcm_dpo/delta": 0.00901147723197937, + "fcm_dpo/margin": 362.3079833984375, + "fcm_dpo/q_t": 0.4271290600299835, + "grad_norm": 35.961185455322266, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.006606936454773, + "logits/rejected": -1.032975435256958, + "logps/chosen": -772.64013671875, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1192.023193359375, + "loss": 1.1563, + "margin_dpo/margin_mean": 362.3079833984375, + "margin_dpo/margin_std": 561.8965454101562, + "step": 588 + }, + { + "KL/chosen_KL_mean": -724.0799560546875, + "KL/mean": -963.3377075195312, + "KL/rejected_KL_mean": -1202.595458984375, + "KL/std": 580.840576171875, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0008570689242333174, + "fcm_dpo/delta": -0.010568767786026001, + "fcm_dpo/margin": 478.51568603515625, + "fcm_dpo/q_t": 0.4057633876800537, + "grad_norm": 33.609222412109375, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.0198304653167725, + "logits/rejected": -1.0384840965270996, + "logps/chosen": -784.0098266601562, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1314.2508544921875, + "loss": 1.0841, + "margin_dpo/margin_mean": 478.5156555175781, + "margin_dpo/margin_std": 625.990478515625, + "step": 589 + }, + { + "KL/chosen_KL_mean": -620.9422607421875, + "KL/mean": -867.03125, + "KL/rejected_KL_mean": -1113.120361328125, + "KL/std": 587.0701904296875, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0008521921699866652, + "fcm_dpo/delta": -0.020320728421211243, + "fcm_dpo/margin": 492.17803955078125, + "fcm_dpo/q_t": 0.40489786863327026, + "grad_norm": 31.491058349609375, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -0.9868652820587158, + "logits/rejected": -1.0203710794448853, + "logps/chosen": -676.7520751953125, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1219.18310546875, + "loss": 1.0786, + "margin_dpo/margin_mean": 492.177978515625, + "margin_dpo/margin_std": 645.5065307617188, + "step": 590 + }, + { + "KL/chosen_KL_mean": -695.0584716796875, + "KL/mean": -915.294921875, + "KL/rejected_KL_mean": -1135.531494140625, + "KL/std": 555.317138671875, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.000855407677590847, + "fcm_dpo/delta": 0.024118637666106224, + "fcm_dpo/margin": 440.47296142578125, + "fcm_dpo/q_t": 0.4125646948814392, + "grad_norm": 36.17401123046875, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.0351864099502563, + "logits/rejected": -1.0366413593292236, + "logps/chosen": -751.299072265625, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1219.3177490234375, + "loss": 1.1141, + "margin_dpo/margin_mean": 440.4729309082031, + "margin_dpo/margin_std": 635.0528564453125, + "step": 591 + }, + { + "KL/chosen_KL_mean": -737.3804931640625, + "KL/mean": -934.9606323242188, + "KL/rejected_KL_mean": -1132.540771484375, + "KL/std": 576.9722900390625, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0008634268888272345, + "fcm_dpo/delta": 0.0609201155602932, + "fcm_dpo/margin": 395.16021728515625, + "fcm_dpo/q_t": 0.4226454198360443, + "grad_norm": 39.21860122680664, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.0073373317718506, + "logits/rejected": -1.012416124343872, + "logps/chosen": -785.32080078125, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1208.273681640625, + "loss": 1.1861, + "margin_dpo/margin_mean": 395.16021728515625, + "margin_dpo/margin_std": 786.0506591796875, + "step": 592 + }, + { + "KL/chosen_KL_mean": -696.8955078125, + "KL/mean": -851.0958862304688, + "KL/rejected_KL_mean": -1005.296142578125, + "KL/std": 601.7348022460938, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.000872555305249989, + "fcm_dpo/delta": 0.03329714015126228, + "fcm_dpo/margin": 308.4005432128906, + "fcm_dpo/q_t": 0.4390355348587036, + "grad_norm": 48.27361297607422, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.8317620754241943, + "logits/rejected": -0.8212481737136841, + "logps/chosen": -745.5863037109375, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1066.2041015625, + "loss": 1.2172, + "margin_dpo/margin_mean": 308.4005432128906, + "margin_dpo/margin_std": 669.50146484375, + "step": 593 + }, + { + "KL/chosen_KL_mean": -669.9156494140625, + "KL/mean": -861.6627197265625, + "KL/rejected_KL_mean": -1053.40966796875, + "KL/std": 591.9979248046875, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.0008697076700627804, + "fcm_dpo/delta": -0.03274283558130264, + "fcm_dpo/margin": 383.49407958984375, + "fcm_dpo/q_t": 0.4233238697052002, + "grad_norm": 46.53880310058594, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -0.9905921220779419, + "logits/rejected": -0.9802975654602051, + "logps/chosen": -724.8504638671875, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1139.5093994140625, + "loss": 1.1572, + "margin_dpo/margin_mean": 383.49407958984375, + "margin_dpo/margin_std": 634.1058349609375, + "step": 594 + }, + { + "KL/chosen_KL_mean": -619.594970703125, + "KL/mean": -837.3834228515625, + "KL/rejected_KL_mean": -1055.1719970703125, + "KL/std": 551.594970703125, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0008683393825776875, + "fcm_dpo/delta": 0.02260814979672432, + "fcm_dpo/margin": 435.57696533203125, + "fcm_dpo/q_t": 0.41516780853271484, + "grad_norm": 46.8239631652832, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -0.9617241024971008, + "logits/rejected": -0.9834858179092407, + "logps/chosen": -669.015380859375, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1135.79931640625, + "loss": 1.1081, + "margin_dpo/margin_mean": 435.5769958496094, + "margin_dpo/margin_std": 614.0455322265625, + "step": 595 + }, + { + "KL/chosen_KL_mean": -714.7357177734375, + "KL/mean": -863.65234375, + "KL/rejected_KL_mean": -1012.56884765625, + "KL/std": 550.900390625, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.0008707884699106216, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 297.833251953125, + "fcm_dpo/q_t": 0.44205644726753235, + "grad_norm": 73.28429412841797, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -0.9765877723693848, + "logits/rejected": -0.9504504799842834, + "logps/chosen": -774.52734375, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1092.6600341796875, + "loss": 1.2214, + "margin_dpo/margin_mean": 297.833251953125, + "margin_dpo/margin_std": 664.2495727539062, + "step": 596 + }, + { + "KL/chosen_KL_mean": -698.87255859375, + "KL/mean": -972.67529296875, + "KL/rejected_KL_mean": -1246.4779052734375, + "KL/std": 658.576904296875, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.000847813964355737, + "fcm_dpo/delta": -0.13368931412696838, + "fcm_dpo/margin": 547.60546875, + "fcm_dpo/q_t": 0.3956410884857178, + "grad_norm": 26.319921493530273, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.9376083016395569, + "logits/rejected": -0.9870784282684326, + "logps/chosen": -756.13330078125, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1347.171630859375, + "loss": 1.0679, + "margin_dpo/margin_mean": 547.60546875, + "margin_dpo/margin_std": 724.9549560546875, + "step": 597 + }, + { + "KL/chosen_KL_mean": -699.13623046875, + "KL/mean": -907.9549560546875, + "KL/rejected_KL_mean": -1116.7738037109375, + "KL/std": 567.6926879882812, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0008396678604185581, + "fcm_dpo/delta": -0.04827401787042618, + "fcm_dpo/margin": 417.63763427734375, + "fcm_dpo/q_t": 0.41797149181365967, + "grad_norm": 35.591773986816406, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.0086700916290283, + "logits/rejected": -1.0213956832885742, + "logps/chosen": -751.6546630859375, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1206.2176513671875, + "loss": 1.1365, + "margin_dpo/margin_mean": 417.6375732421875, + "margin_dpo/margin_std": 629.5519409179688, + "step": 598 + }, + { + "KL/chosen_KL_mean": -711.6976928710938, + "KL/mean": -888.90625, + "KL/rejected_KL_mean": -1066.11474609375, + "KL/std": 557.2623901367188, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0008341920329257846, + "fcm_dpo/delta": -0.03271396830677986, + "fcm_dpo/margin": 354.4170837402344, + "fcm_dpo/q_t": 0.42789605259895325, + "grad_norm": 34.62250900268555, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.9513130784034729, + "logits/rejected": -0.9624460339546204, + "logps/chosen": -761.5003662109375, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1149.09326171875, + "loss": 1.1656, + "margin_dpo/margin_mean": 354.4170837402344, + "margin_dpo/margin_std": 533.1595458984375, + "step": 599 + }, + { + "KL/chosen_KL_mean": -755.8899536132812, + "KL/mean": -927.88720703125, + "KL/rejected_KL_mean": -1099.88427734375, + "KL/std": 568.3023681640625, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.000840198656078428, + "fcm_dpo/delta": 0.03587355837225914, + "fcm_dpo/margin": 343.99432373046875, + "fcm_dpo/q_t": 0.4352074861526489, + "grad_norm": 34.23080825805664, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.0457412004470825, + "logits/rejected": -1.046329379081726, + "logps/chosen": -822.3248291015625, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1185.3408203125, + "loss": 1.198, + "margin_dpo/margin_mean": 343.99432373046875, + "margin_dpo/margin_std": 700.41796875, + "step": 600 + }, + { + "KL/chosen_KL_mean": -771.788818359375, + "KL/mean": -989.244384765625, + "KL/rejected_KL_mean": -1206.699951171875, + "KL/std": 595.355712890625, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0008434988558292389, + "fcm_dpo/delta": 0.034427203238010406, + "fcm_dpo/margin": 434.9111328125, + "fcm_dpo/q_t": 0.4143131375312805, + "grad_norm": 35.2708854675293, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -0.9935545921325684, + "logits/rejected": -1.0154894590377808, + "logps/chosen": -830.9224243164062, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1301.390869140625, + "loss": 1.1106, + "margin_dpo/margin_mean": 434.9111328125, + "margin_dpo/margin_std": 586.5540771484375, + "step": 601 + }, + { + "KL/chosen_KL_mean": -534.0012817382812, + "KL/mean": -801.1100463867188, + "KL/rejected_KL_mean": -1068.21875, + "KL/std": 550.30908203125, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0008439007215201855, + "fcm_dpo/delta": -0.05354182422161102, + "fcm_dpo/margin": 534.217529296875, + "fcm_dpo/q_t": 0.396359384059906, + "grad_norm": 77.93232727050781, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -1.008573055267334, + "logits/rejected": -1.0463042259216309, + "logps/chosen": -582.5947875976562, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1155.8873291015625, + "loss": 1.0459, + "margin_dpo/margin_mean": 534.217529296875, + "margin_dpo/margin_std": 599.2060546875, + "step": 602 + }, + { + "KL/chosen_KL_mean": -669.5146484375, + "KL/mean": -911.340087890625, + "KL/rejected_KL_mean": -1153.16552734375, + "KL/std": 607.0291137695312, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0008357585174962878, + "fcm_dpo/delta": -0.0044101811945438385, + "fcm_dpo/margin": 483.6508483886719, + "fcm_dpo/q_t": 0.4096281826496124, + "grad_norm": 40.72458267211914, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -1.0025463104248047, + "logits/rejected": -0.9986417293548584, + "logps/chosen": -739.9292602539062, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1253.490966796875, + "loss": 1.1028, + "margin_dpo/margin_mean": 483.6508483886719, + "margin_dpo/margin_std": 714.4725952148438, + "step": 603 + }, + { + "KL/chosen_KL_mean": -653.669189453125, + "KL/mean": -916.517333984375, + "KL/rejected_KL_mean": -1179.3653564453125, + "KL/std": 597.5855712890625, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0008314626757055521, + "fcm_dpo/delta": -0.03879137337207794, + "fcm_dpo/margin": 525.6961669921875, + "fcm_dpo/q_t": 0.4024570882320404, + "grad_norm": 37.43436050415039, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -0.9756330847740173, + "logits/rejected": -1.0146242380142212, + "logps/chosen": -700.1273193359375, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1271.219970703125, + "loss": 1.0839, + "margin_dpo/margin_mean": 525.6961669921875, + "margin_dpo/margin_std": 746.1463012695312, + "step": 604 + }, + { + "KL/chosen_KL_mean": -696.1136474609375, + "KL/mean": -885.1508178710938, + "KL/rejected_KL_mean": -1074.18798828125, + "KL/std": 557.2295532226562, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0008325940580107272, + "fcm_dpo/delta": 0.016855284571647644, + "fcm_dpo/margin": 378.07427978515625, + "fcm_dpo/q_t": 0.42936962842941284, + "grad_norm": 40.84849548339844, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -0.9307333827018738, + "logits/rejected": -0.9448376893997192, + "logps/chosen": -762.3629760742188, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1176.492919921875, + "loss": 1.1665, + "margin_dpo/margin_mean": 378.07427978515625, + "margin_dpo/margin_std": 670.087158203125, + "step": 605 + }, + { + "KL/chosen_KL_mean": -682.961669921875, + "KL/mean": -895.4780883789062, + "KL/rejected_KL_mean": -1107.99462890625, + "KL/std": 563.1690673828125, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0008319264743477106, + "fcm_dpo/delta": -0.00802488811314106, + "fcm_dpo/margin": 425.032958984375, + "fcm_dpo/q_t": 0.41894659399986267, + "grad_norm": 25.7265625, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -0.9446300268173218, + "logits/rejected": -0.9631451368331909, + "logps/chosen": -737.78076171875, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1206.365966796875, + "loss": 1.1235, + "margin_dpo/margin_mean": 425.032958984375, + "margin_dpo/margin_std": 609.4775390625, + "step": 606 + }, + { + "KL/chosen_KL_mean": -704.001220703125, + "KL/mean": -880.4382934570312, + "KL/rejected_KL_mean": -1056.87548828125, + "KL/std": 564.08056640625, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0008335023303516209, + "fcm_dpo/delta": 0.02691652812063694, + "fcm_dpo/margin": 352.87420654296875, + "fcm_dpo/q_t": 0.4321540594100952, + "grad_norm": 25.09073829650879, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.9357039928436279, + "logits/rejected": -0.927735447883606, + "logps/chosen": -762.085205078125, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1136.652587890625, + "loss": 1.1776, + "margin_dpo/margin_mean": 352.8742370605469, + "margin_dpo/margin_std": 619.4921875, + "step": 607 + }, + { + "KL/chosen_KL_mean": -638.0068359375, + "KL/mean": -844.9004516601562, + "KL/rejected_KL_mean": -1051.794189453125, + "KL/std": 521.3062133789062, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.0008442871039733291, + "fcm_dpo/delta": 0.05223855748772621, + "fcm_dpo/margin": 413.7873229980469, + "fcm_dpo/q_t": 0.4184677302837372, + "grad_norm": 37.57941436767578, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -0.9944363236427307, + "logits/rejected": -1.0088995695114136, + "logps/chosen": -695.4576416015625, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1146.5675048828125, + "loss": 1.1176, + "margin_dpo/margin_mean": 413.78729248046875, + "margin_dpo/margin_std": 545.526123046875, + "step": 608 + }, + { + "KL/chosen_KL_mean": -651.825927734375, + "KL/mean": -919.9029541015625, + "KL/rejected_KL_mean": -1187.97998046875, + "KL/std": 694.5575561523438, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.000838621286675334, + "fcm_dpo/delta": -0.05194016546010971, + "fcm_dpo/margin": 536.154052734375, + "fcm_dpo/q_t": 0.4039934277534485, + "grad_norm": 39.401424407958984, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.0233325958251953, + "logits/rejected": -1.0491836071014404, + "logps/chosen": -710.6312255859375, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1276.7958984375, + "loss": 1.0875, + "margin_dpo/margin_mean": 536.154052734375, + "margin_dpo/margin_std": 842.8841552734375, + "step": 609 + }, + { + "KL/chosen_KL_mean": -652.326416015625, + "KL/mean": -820.6024169921875, + "KL/rejected_KL_mean": -988.8783569335938, + "KL/std": 538.6533203125, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0008393626776523888, + "fcm_dpo/delta": 0.04249637946486473, + "fcm_dpo/margin": 336.5518798828125, + "fcm_dpo/q_t": 0.4335845112800598, + "grad_norm": 40.75104522705078, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -0.9901823997497559, + "logits/rejected": -0.9753342866897583, + "logps/chosen": -718.021484375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1072.28369140625, + "loss": 1.191, + "margin_dpo/margin_mean": 336.5518798828125, + "margin_dpo/margin_std": 637.683837890625, + "step": 610 + }, + { + "KL/chosen_KL_mean": -699.4143676757812, + "KL/mean": -985.2313232421875, + "KL/rejected_KL_mean": -1271.04833984375, + "KL/std": 706.6036376953125, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0008329330012202263, + "fcm_dpo/delta": -0.07994256168603897, + "fcm_dpo/margin": 571.634033203125, + "fcm_dpo/q_t": 0.3955478072166443, + "grad_norm": 32.484703063964844, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.0565005540847778, + "logits/rejected": -1.1088647842407227, + "logps/chosen": -752.0137939453125, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1357.37939453125, + "loss": 1.0671, + "margin_dpo/margin_mean": 571.634033203125, + "margin_dpo/margin_std": 811.6180419921875, + "step": 611 + }, + { + "KL/chosen_KL_mean": -746.6201782226562, + "KL/mean": -963.3016967773438, + "KL/rejected_KL_mean": -1179.983154296875, + "KL/std": 568.2952270507812, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0008285264484584332, + "fcm_dpo/delta": -0.01247065793722868, + "fcm_dpo/margin": 433.3630065917969, + "fcm_dpo/q_t": 0.41694575548171997, + "grad_norm": 29.30278968811035, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -1.0173687934875488, + "logits/rejected": -1.029083490371704, + "logps/chosen": -805.9439086914062, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1268.295654296875, + "loss": 1.1209, + "margin_dpo/margin_mean": 433.36297607421875, + "margin_dpo/margin_std": 615.6133422851562, + "step": 612 + }, + { + "KL/chosen_KL_mean": -640.5697631835938, + "KL/mean": -908.50244140625, + "KL/rejected_KL_mean": -1176.43505859375, + "KL/std": 624.0267333984375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0008206545608118176, + "fcm_dpo/delta": -0.041764743626117706, + "fcm_dpo/margin": 535.8653564453125, + "fcm_dpo/q_t": 0.3986205458641052, + "grad_norm": 40.39195251464844, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -0.9570515155792236, + "logits/rejected": -1.0131473541259766, + "logps/chosen": -700.2996826171875, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1281.5426025390625, + "loss": 1.0586, + "margin_dpo/margin_mean": 535.8653564453125, + "margin_dpo/margin_std": 644.5519409179688, + "step": 613 + }, + { + "KL/chosen_KL_mean": -715.4906005859375, + "KL/mean": -983.7789916992188, + "KL/rejected_KL_mean": -1252.0673828125, + "KL/std": 634.355224609375, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0008170532528311014, + "fcm_dpo/delta": -0.04016195237636566, + "fcm_dpo/margin": 536.5768432617188, + "fcm_dpo/q_t": 0.4015321135520935, + "grad_norm": 45.062320709228516, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -0.9467099905014038, + "logits/rejected": -1.0279295444488525, + "logps/chosen": -768.4295654296875, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1356.746826171875, + "loss": 1.0801, + "margin_dpo/margin_mean": 536.5768432617188, + "margin_dpo/margin_std": 750.0596923828125, + "step": 614 + }, + { + "KL/chosen_KL_mean": -678.5205688476562, + "KL/mean": -895.8533935546875, + "KL/rejected_KL_mean": -1113.186279296875, + "KL/std": 649.1279296875, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0008215580601245165, + "fcm_dpo/delta": 0.04423396289348602, + "fcm_dpo/margin": 434.66571044921875, + "fcm_dpo/q_t": 0.41857171058654785, + "grad_norm": 28.190874099731445, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.9696630835533142, + "logits/rejected": -0.9963263273239136, + "logps/chosen": -744.337890625, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1208.36376953125, + "loss": 1.1375, + "margin_dpo/margin_mean": 434.66571044921875, + "margin_dpo/margin_std": 703.9349365234375, + "step": 615 + }, + { + "KL/chosen_KL_mean": -805.335693359375, + "KL/mean": -974.939697265625, + "KL/rejected_KL_mean": -1144.543701171875, + "KL/std": 538.217529296875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.00082127854693681, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 339.2080383300781, + "fcm_dpo/q_t": 0.4350740313529968, + "grad_norm": 30.26882553100586, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.0494017601013184, + "logits/rejected": -1.0350136756896973, + "logps/chosen": -870.468505859375, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1219.244140625, + "loss": 1.1851, + "margin_dpo/margin_mean": 339.2080383300781, + "margin_dpo/margin_std": 596.009033203125, + "step": 616 + }, + { + "KL/chosen_KL_mean": -773.4424438476562, + "KL/mean": -899.0516357421875, + "KL/rejected_KL_mean": -1024.660888671875, + "KL/std": 515.9038696289062, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.00082127854693681, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 251.21832275390625, + "fcm_dpo/q_t": 0.4523277282714844, + "grad_norm": 61.21628952026367, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.9722844362258911, + "logits/rejected": -0.9445664882659912, + "logps/chosen": -836.447998046875, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1088.8948974609375, + "loss": 1.2537, + "margin_dpo/margin_mean": 251.21832275390625, + "margin_dpo/margin_std": 622.1917114257812, + "step": 617 + }, + { + "KL/chosen_KL_mean": -819.5065307617188, + "KL/mean": -1090.12646484375, + "KL/rejected_KL_mean": -1360.746337890625, + "KL/std": 694.7017822265625, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0008206435013562441, + "fcm_dpo/delta": -0.04670292139053345, + "fcm_dpo/margin": 541.23974609375, + "fcm_dpo/q_t": 0.4022930860519409, + "grad_norm": 37.068077087402344, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -1.0099928379058838, + "logits/rejected": -1.0500186681747437, + "logps/chosen": -886.60791015625, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1452.899658203125, + "loss": 1.1154, + "margin_dpo/margin_mean": 541.23974609375, + "margin_dpo/margin_std": 901.9833374023438, + "step": 618 + }, + { + "KL/chosen_KL_mean": -792.280517578125, + "KL/mean": -997.8952026367188, + "KL/rejected_KL_mean": -1203.509765625, + "KL/std": 653.9510498046875, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0008158984710462391, + "fcm_dpo/delta": 0.027643609791994095, + "fcm_dpo/margin": 411.2293701171875, + "fcm_dpo/q_t": 0.4292343258857727, + "grad_norm": 45.077880859375, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -0.9693220853805542, + "logits/rejected": -0.9858847260475159, + "logps/chosen": -848.2587280273438, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1296.6953125, + "loss": 1.1944, + "margin_dpo/margin_mean": 411.2293701171875, + "margin_dpo/margin_std": 878.142578125, + "step": 619 + }, + { + "KL/chosen_KL_mean": -712.3096923828125, + "KL/mean": -900.9335327148438, + "KL/rejected_KL_mean": -1089.557373046875, + "KL/std": 569.0059814453125, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.000819763052277267, + "fcm_dpo/delta": 0.0098243597894907, + "fcm_dpo/margin": 377.2477722167969, + "fcm_dpo/q_t": 0.4286845922470093, + "grad_norm": 42.07814407348633, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.0219039916992188, + "logits/rejected": -1.0302460193634033, + "logps/chosen": -772.107177734375, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1167.9681396484375, + "loss": 1.1652, + "margin_dpo/margin_mean": 377.24774169921875, + "margin_dpo/margin_std": 634.4324340820312, + "step": 620 + }, + { + "KL/chosen_KL_mean": -713.438232421875, + "KL/mean": -1016.79248046875, + "KL/rejected_KL_mean": -1320.146484375, + "KL/std": 701.0230712890625, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.0008137230761349201, + "fcm_dpo/delta": -0.0989176332950592, + "fcm_dpo/margin": 606.708251953125, + "fcm_dpo/q_t": 0.3883476257324219, + "grad_norm": 47.212833404541016, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.061957836151123, + "logits/rejected": -1.1169843673706055, + "logps/chosen": -767.3720703125, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1408.51611328125, + "loss": 1.0481, + "margin_dpo/margin_mean": 606.708251953125, + "margin_dpo/margin_std": 780.872314453125, + "step": 621 + }, + { + "KL/chosen_KL_mean": -704.105224609375, + "KL/mean": -914.0382080078125, + "KL/rejected_KL_mean": -1123.97119140625, + "KL/std": 542.4420166015625, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0008017644286155701, + "fcm_dpo/delta": -0.012085220776498318, + "fcm_dpo/margin": 419.8658447265625, + "fcm_dpo/q_t": 0.4236697554588318, + "grad_norm": 34.02726745605469, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.930920422077179, + "logits/rejected": -0.9223548769950867, + "logps/chosen": -764.39111328125, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1209.4898681640625, + "loss": 1.1453, + "margin_dpo/margin_mean": 419.8658447265625, + "margin_dpo/margin_std": 681.1514892578125, + "step": 622 + }, + { + "KL/chosen_KL_mean": -749.1619873046875, + "KL/mean": -910.14794921875, + "KL/rejected_KL_mean": -1071.134033203125, + "KL/std": 544.9346313476562, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0008017644286155701, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 321.97198486328125, + "fcm_dpo/q_t": 0.44129130244255066, + "grad_norm": 37.289939880371094, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.0920281410217285, + "logits/rejected": -1.0948947668075562, + "logps/chosen": -813.3189697265625, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1156.217041015625, + "loss": 1.2093, + "margin_dpo/margin_mean": 321.97198486328125, + "margin_dpo/margin_std": 657.8831787109375, + "step": 623 + }, + { + "KL/chosen_KL_mean": -753.6048583984375, + "KL/mean": -1010.7816162109375, + "KL/rejected_KL_mean": -1267.958251953125, + "KL/std": 576.8868408203125, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0008008379372768104, + "fcm_dpo/delta": -0.012426842004060745, + "fcm_dpo/margin": 514.353515625, + "fcm_dpo/q_t": 0.40316253900527954, + "grad_norm": 41.44575500488281, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.0857577323913574, + "logits/rejected": -1.0974596738815308, + "logps/chosen": -825.5234985351562, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1365.09033203125, + "loss": 1.0758, + "margin_dpo/margin_mean": 514.353515625, + "margin_dpo/margin_std": 624.5205078125, + "step": 624 + }, + { + "KL/chosen_KL_mean": -727.935546875, + "KL/mean": -1007.9580078125, + "KL/rejected_KL_mean": -1287.98046875, + "KL/std": 645.153076171875, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0007958546630106866, + "fcm_dpo/delta": -0.04784073308110237, + "fcm_dpo/margin": 560.0449829101562, + "fcm_dpo/q_t": 0.3969777822494507, + "grad_norm": 61.39133834838867, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -0.9978440999984741, + "logits/rejected": -1.0282714366912842, + "logps/chosen": -786.277587890625, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1374.0709228515625, + "loss": 1.0414, + "margin_dpo/margin_mean": 560.0449829101562, + "margin_dpo/margin_std": 600.5526733398438, + "step": 625 + }, + { + "KL/chosen_KL_mean": -870.1897583007812, + "KL/mean": -1028.6336669921875, + "KL/rejected_KL_mean": -1187.07763671875, + "KL/std": 684.579833984375, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0007921583601273596, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 316.88787841796875, + "fcm_dpo/q_t": 0.4400758743286133, + "grad_norm": 40.44783401489258, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -0.9589856863021851, + "logits/rejected": -0.9511022567749023, + "logps/chosen": -945.3023681640625, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1286.266357421875, + "loss": 1.2214, + "margin_dpo/margin_mean": 316.88787841796875, + "margin_dpo/margin_std": 669.0109252929688, + "step": 626 + }, + { + "KL/chosen_KL_mean": -593.00244140625, + "KL/mean": -900.6890869140625, + "KL/rejected_KL_mean": -1208.3756103515625, + "KL/std": 745.9805908203125, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0007866608793847263, + "fcm_dpo/delta": -0.08859986066818237, + "fcm_dpo/margin": 615.3731689453125, + "fcm_dpo/q_t": 0.3975117802619934, + "grad_norm": 27.381633758544922, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.9167486429214478, + "logits/rejected": -0.9933542013168335, + "logps/chosen": -640.74560546875, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1315.130126953125, + "loss": 1.0632, + "margin_dpo/margin_mean": 615.3731689453125, + "margin_dpo/margin_std": 862.61181640625, + "step": 627 + }, + { + "KL/chosen_KL_mean": -801.4574584960938, + "KL/mean": -1032.044677734375, + "KL/rejected_KL_mean": -1262.632080078125, + "KL/std": 622.4959716796875, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0007823093910701573, + "fcm_dpo/delta": 0.04070672392845154, + "fcm_dpo/margin": 461.17449951171875, + "fcm_dpo/q_t": 0.4180537462234497, + "grad_norm": 31.138437271118164, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.0785002708435059, + "logits/rejected": -1.0994800329208374, + "logps/chosen": -861.640380859375, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1364.1866455078125, + "loss": 1.1324, + "margin_dpo/margin_mean": 461.17449951171875, + "margin_dpo/margin_std": 720.4835205078125, + "step": 628 + }, + { + "KL/chosen_KL_mean": -760.44482421875, + "KL/mean": -971.9022216796875, + "KL/rejected_KL_mean": -1183.3597412109375, + "KL/std": 589.089111328125, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0007925507379695773, + "fcm_dpo/delta": 0.0670618936419487, + "fcm_dpo/margin": 422.9149169921875, + "fcm_dpo/q_t": 0.421453058719635, + "grad_norm": 32.154483795166016, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.042360782623291, + "logits/rejected": -1.0599124431610107, + "logps/chosen": -824.6583251953125, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1275.013427734375, + "loss": 1.1315, + "margin_dpo/margin_mean": 422.9149169921875, + "margin_dpo/margin_std": 592.815673828125, + "step": 629 + }, + { + "KL/chosen_KL_mean": -720.1742553710938, + "KL/mean": -844.034423828125, + "KL/rejected_KL_mean": -967.8946533203125, + "KL/std": 605.764404296875, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0007952011073939502, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 247.72032165527344, + "fcm_dpo/q_t": 0.45805439352989197, + "grad_norm": 61.329071044921875, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -1.017009973526001, + "logits/rejected": -0.991877555847168, + "logps/chosen": -779.4652709960938, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1051.492919921875, + "loss": 1.2796, + "margin_dpo/margin_mean": 247.72032165527344, + "margin_dpo/margin_std": 736.06494140625, + "step": 630 + }, + { + "KL/chosen_KL_mean": -844.1835327148438, + "KL/mean": -1038.1522216796875, + "KL/rejected_KL_mean": -1232.120849609375, + "KL/std": 609.7965087890625, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0007981567177921534, + "fcm_dpo/delta": 0.03703029453754425, + "fcm_dpo/margin": 387.9373779296875, + "fcm_dpo/q_t": 0.4280385375022888, + "grad_norm": 30.539676666259766, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.1132234334945679, + "logits/rejected": -1.1241021156311035, + "logps/chosen": -903.6371459960938, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1313.072509765625, + "loss": 1.1778, + "margin_dpo/margin_mean": 387.9373779296875, + "margin_dpo/margin_std": 714.165771484375, + "step": 631 + }, + { + "KL/chosen_KL_mean": -739.439697265625, + "KL/mean": -940.5237426757812, + "KL/rejected_KL_mean": -1141.60791015625, + "KL/std": 560.5201416015625, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.0008009191369637847, + "fcm_dpo/delta": -0.0012054782127961516, + "fcm_dpo/margin": 402.1682434082031, + "fcm_dpo/q_t": 0.4253075122833252, + "grad_norm": 42.51784133911133, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.059547781944275, + "logits/rejected": -1.0634690523147583, + "logps/chosen": -800.7911987304688, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1227.76806640625, + "loss": 1.1643, + "margin_dpo/margin_mean": 402.1682434082031, + "margin_dpo/margin_std": 693.4299926757812, + "step": 632 + }, + { + "KL/chosen_KL_mean": -821.0465087890625, + "KL/mean": -968.1879272460938, + "KL/rejected_KL_mean": -1115.329345703125, + "KL/std": 567.065185546875, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0008009191369637847, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 294.28277587890625, + "fcm_dpo/q_t": 0.4446948170661926, + "grad_norm": 49.53974914550781, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.0918877124786377, + "logits/rejected": -1.102707862854004, + "logps/chosen": -878.32470703125, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1206.913330078125, + "loss": 1.2287, + "margin_dpo/margin_mean": 294.2828063964844, + "margin_dpo/margin_std": 644.408447265625, + "step": 633 + }, + { + "KL/chosen_KL_mean": -918.9446411132812, + "KL/mean": -1050.765625, + "KL/rejected_KL_mean": -1182.5865478515625, + "KL/std": 658.0369262695312, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0008009191369637847, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 263.6419372558594, + "fcm_dpo/q_t": 0.451707124710083, + "grad_norm": 33.28659439086914, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.9775643348693848, + "logits/rejected": -0.9735915064811707, + "logps/chosen": -985.5635986328125, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1289.712158203125, + "loss": 1.2494, + "margin_dpo/margin_mean": 263.6419372558594, + "margin_dpo/margin_std": 643.177978515625, + "step": 634 + }, + { + "KL/chosen_KL_mean": -778.28515625, + "KL/mean": -960.452392578125, + "KL/rejected_KL_mean": -1142.61962890625, + "KL/std": 686.8685913085938, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.0008009191369637847, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 364.33447265625, + "fcm_dpo/q_t": 0.4342397153377533, + "grad_norm": 39.356563568115234, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.0562341213226318, + "logits/rejected": -1.0381475687026978, + "logps/chosen": -852.24072265625, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1225.1201171875, + "loss": 1.1868, + "margin_dpo/margin_mean": 364.33447265625, + "margin_dpo/margin_std": 698.2109985351562, + "step": 635 + }, + { + "KL/chosen_KL_mean": -742.5096435546875, + "KL/mean": -962.7926025390625, + "KL/rejected_KL_mean": -1183.07568359375, + "KL/std": 660.0510864257812, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0007981258677318692, + "fcm_dpo/delta": -0.03499758988618851, + "fcm_dpo/margin": 440.5659484863281, + "fcm_dpo/q_t": 0.420836478471756, + "grad_norm": 27.303789138793945, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.9779366254806519, + "logits/rejected": -0.981173038482666, + "logps/chosen": -802.1385498046875, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1265.054443359375, + "loss": 1.1335, + "margin_dpo/margin_mean": 440.56597900390625, + "margin_dpo/margin_std": 689.0616455078125, + "step": 636 + }, + { + "KL/chosen_KL_mean": -735.320068359375, + "KL/mean": -987.8580932617188, + "KL/rejected_KL_mean": -1240.3961181640625, + "KL/std": 673.1341552734375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0007922681979835033, + "fcm_dpo/delta": -0.000458642840385437, + "fcm_dpo/margin": 505.0760498046875, + "fcm_dpo/q_t": 0.4097414016723633, + "grad_norm": 35.437625885009766, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.0780822038650513, + "logits/rejected": -1.1320710182189941, + "logps/chosen": -784.9727172851562, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1338.80126953125, + "loss": 1.0979, + "margin_dpo/margin_mean": 505.0760498046875, + "margin_dpo/margin_std": 709.5736083984375, + "step": 637 + }, + { + "KL/chosen_KL_mean": -726.78271484375, + "KL/mean": -921.652099609375, + "KL/rejected_KL_mean": -1116.5216064453125, + "KL/std": 626.0118408203125, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0007920425850898027, + "fcm_dpo/delta": -0.04061838984489441, + "fcm_dpo/margin": 389.7388610839844, + "fcm_dpo/q_t": 0.42422229051589966, + "grad_norm": 30.393226623535156, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.9700440764427185, + "logits/rejected": -0.955012321472168, + "logps/chosen": -784.9393920898438, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1195.822998046875, + "loss": 1.1861, + "margin_dpo/margin_mean": 389.7388916015625, + "margin_dpo/margin_std": 734.9174194335938, + "step": 638 + }, + { + "KL/chosen_KL_mean": -996.0430908203125, + "KL/mean": -1078.2799072265625, + "KL/rejected_KL_mean": -1160.5166015625, + "KL/std": 593.8790893554688, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0007888254476711154, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 164.47360229492188, + "fcm_dpo/q_t": 0.47027307748794556, + "grad_norm": 113.94747924804688, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.9586646556854248, + "logits/rejected": -0.9000857472419739, + "logps/chosen": -1068.3662109375, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1234.79150390625, + "loss": 1.3435, + "margin_dpo/margin_mean": 164.47360229492188, + "margin_dpo/margin_std": 743.249755859375, + "step": 639 + }, + { + "KL/chosen_KL_mean": -762.9091186523438, + "KL/mean": -1054.9638671875, + "KL/rejected_KL_mean": -1347.0186767578125, + "KL/std": 701.1949462890625, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.0007847544620744884, + "fcm_dpo/delta": -0.0612616091966629, + "fcm_dpo/margin": 584.1094970703125, + "fcm_dpo/q_t": 0.3992745280265808, + "grad_norm": 48.65264129638672, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.9705928564071655, + "logits/rejected": -1.0060577392578125, + "logps/chosen": -819.04345703125, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1455.6187744140625, + "loss": 1.0743, + "margin_dpo/margin_mean": 584.1094970703125, + "margin_dpo/margin_std": 832.4061279296875, + "step": 640 + }, + { + "KL/chosen_KL_mean": -883.6129760742188, + "KL/mean": -1079.036376953125, + "KL/rejected_KL_mean": -1274.459716796875, + "KL/std": 611.6572875976562, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0007849803077988327, + "fcm_dpo/delta": 0.03682980313897133, + "fcm_dpo/margin": 390.8468017578125, + "fcm_dpo/q_t": 0.4321765899658203, + "grad_norm": 27.26753807067871, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.0182573795318604, + "logits/rejected": -1.0299720764160156, + "logps/chosen": -948.60986328125, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1361.4521484375, + "loss": 1.1986, + "margin_dpo/margin_mean": 390.8468017578125, + "margin_dpo/margin_std": 828.2184448242188, + "step": 641 + }, + { + "KL/chosen_KL_mean": -861.0926513671875, + "KL/mean": -1099.9857177734375, + "KL/rejected_KL_mean": -1338.8787841796875, + "KL/std": 812.6694946289062, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.000782849732786417, + "fcm_dpo/delta": -0.02721548266708851, + "fcm_dpo/margin": 477.78619384765625, + "fcm_dpo/q_t": 0.4226352572441101, + "grad_norm": 43.7198486328125, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -0.9503949880599976, + "logits/rejected": -0.9886642694473267, + "logps/chosen": -926.7818603515625, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1449.120849609375, + "loss": 1.1702, + "margin_dpo/margin_mean": 477.7862243652344, + "margin_dpo/margin_std": 982.9949951171875, + "step": 642 + }, + { + "KL/chosen_KL_mean": -757.2095336914062, + "KL/mean": -937.7981567382812, + "KL/rejected_KL_mean": -1118.386962890625, + "KL/std": 583.7869873046875, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0007799399900250137, + "fcm_dpo/delta": -0.004992997739464045, + "fcm_dpo/margin": 361.1773681640625, + "fcm_dpo/q_t": 0.4340949058532715, + "grad_norm": 39.68205642700195, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.0948643684387207, + "logits/rejected": -1.0938928127288818, + "logps/chosen": -809.1595458984375, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1205.855224609375, + "loss": 1.1774, + "margin_dpo/margin_mean": 361.1773681640625, + "margin_dpo/margin_std": 582.2108154296875, + "step": 643 + }, + { + "KL/chosen_KL_mean": -752.95068359375, + "KL/mean": -935.5848388671875, + "KL/rejected_KL_mean": -1118.218994140625, + "KL/std": 681.5095825195312, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0007799399900250137, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 365.26837158203125, + "fcm_dpo/q_t": 0.4356444180011749, + "grad_norm": 58.357791900634766, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.0311143398284912, + "logits/rejected": -1.039900779724121, + "logps/chosen": -811.9683227539062, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1205.355712890625, + "loss": 1.2121, + "margin_dpo/margin_mean": 365.2683410644531, + "margin_dpo/margin_std": 789.8173217773438, + "step": 644 + }, + { + "KL/chosen_KL_mean": -782.1712646484375, + "KL/mean": -873.3842163085938, + "KL/rejected_KL_mean": -964.59716796875, + "KL/std": 532.8189086914062, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0007799399900250137, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 182.42578125, + "fcm_dpo/q_t": 0.46802952885627747, + "grad_norm": 105.71790313720703, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.972783088684082, + "logits/rejected": -0.9613343477249146, + "logps/chosen": -838.04736328125, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1062.3779296875, + "loss": 1.3326, + "margin_dpo/margin_mean": 182.42579650878906, + "margin_dpo/margin_std": 727.2509765625, + "step": 645 + }, + { + "KL/chosen_KL_mean": -736.2886962890625, + "KL/mean": -893.9510498046875, + "KL/rejected_KL_mean": -1051.61328125, + "KL/std": 512.5478515625, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0007799399900250137, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 315.32452392578125, + "fcm_dpo/q_t": 0.4425877630710602, + "grad_norm": 49.94985580444336, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.0099034309387207, + "logits/rejected": -0.9940212965011597, + "logps/chosen": -797.5645141601562, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1129.119140625, + "loss": 1.2128, + "margin_dpo/margin_mean": 315.32452392578125, + "margin_dpo/margin_std": 615.5316162109375, + "step": 646 + }, + { + "KL/chosen_KL_mean": -649.4276733398438, + "KL/mean": -854.7105712890625, + "KL/rejected_KL_mean": -1059.993408203125, + "KL/std": 584.4964599609375, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0007895968738012016, + "fcm_dpo/delta": 0.0783025249838829, + "fcm_dpo/margin": 410.5657653808594, + "fcm_dpo/q_t": 0.42436790466308594, + "grad_norm": 29.25990104675293, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.9164397120475769, + "logits/rejected": -0.9481786489486694, + "logps/chosen": -704.2800903320312, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1153.5128173828125, + "loss": 1.1408, + "margin_dpo/margin_mean": 410.5657958984375, + "margin_dpo/margin_std": 584.1629638671875, + "step": 647 + }, + { + "KL/chosen_KL_mean": -709.5036010742188, + "KL/mean": -1008.3509521484375, + "KL/rejected_KL_mean": -1307.1982421875, + "KL/std": 713.9449462890625, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.0007842861232347786, + "fcm_dpo/delta": -0.07210898399353027, + "fcm_dpo/margin": 597.69482421875, + "fcm_dpo/q_t": 0.39762264490127563, + "grad_norm": 23.83572769165039, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.061322569847107, + "logits/rejected": -1.1233322620391846, + "logps/chosen": -763.675048828125, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1405.9111328125, + "loss": 1.0678, + "margin_dpo/margin_mean": 597.69482421875, + "margin_dpo/margin_std": 856.947021484375, + "step": 648 + }, + { + "KL/chosen_KL_mean": -746.2015380859375, + "KL/mean": -888.7406005859375, + "KL/rejected_KL_mean": -1031.279541015625, + "KL/std": 563.6659545898438, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0007809067028574646, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 285.07794189453125, + "fcm_dpo/q_t": 0.45108920335769653, + "grad_norm": 33.67908477783203, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.088302731513977, + "logits/rejected": -1.082716703414917, + "logps/chosen": -808.681884765625, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1111.356689453125, + "loss": 1.2498, + "margin_dpo/margin_mean": 285.07794189453125, + "margin_dpo/margin_std": 695.422607421875, + "step": 649 + }, + { + "KL/chosen_KL_mean": -791.5379028320312, + "KL/mean": -1013.7835693359375, + "KL/rejected_KL_mean": -1236.029296875, + "KL/std": 696.5599365234375, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0007855009753257036, + "fcm_dpo/delta": 0.05271507054567337, + "fcm_dpo/margin": 444.49139404296875, + "fcm_dpo/q_t": 0.4219529330730438, + "grad_norm": 34.624855041503906, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.0349628925323486, + "logits/rejected": -1.0608723163604736, + "logps/chosen": -847.6307373046875, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1334.294189453125, + "loss": 1.1586, + "margin_dpo/margin_mean": 444.4914245605469, + "margin_dpo/margin_std": 800.30224609375, + "step": 650 + }, + { + "KL/chosen_KL_mean": -530.2730712890625, + "KL/mean": -820.2999267578125, + "KL/rejected_KL_mean": -1110.326904296875, + "KL/std": 638.31005859375, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0007822200423106551, + "fcm_dpo/delta": -0.056315451860427856, + "fcm_dpo/margin": 580.0538330078125, + "fcm_dpo/q_t": 0.395839661359787, + "grad_norm": 40.45621871948242, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.0222262144088745, + "logits/rejected": -1.0831019878387451, + "logps/chosen": -573.698486328125, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1210.28466796875, + "loss": 1.0426, + "margin_dpo/margin_mean": 580.0538330078125, + "margin_dpo/margin_std": 656.2584838867188, + "step": 651 + }, + { + "KL/chosen_KL_mean": -689.8887939453125, + "KL/mean": -908.66552734375, + "KL/rejected_KL_mean": -1127.442138671875, + "KL/std": 657.2205200195312, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.0007882657810114324, + "fcm_dpo/delta": 0.05691511929035187, + "fcm_dpo/margin": 437.5533447265625, + "fcm_dpo/q_t": 0.4222105145454407, + "grad_norm": 34.00590133666992, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.0584585666656494, + "logits/rejected": -1.0897393226623535, + "logps/chosen": -752.4656372070312, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1239.2099609375, + "loss": 1.1535, + "margin_dpo/margin_mean": 437.5533142089844, + "margin_dpo/margin_std": 762.7130126953125, + "step": 652 + }, + { + "KL/chosen_KL_mean": -812.3721313476562, + "KL/mean": -1018.362060546875, + "KL/rejected_KL_mean": -1224.3519287109375, + "KL/std": 669.9310302734375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0007916088216006756, + "fcm_dpo/delta": 0.029443640261888504, + "fcm_dpo/margin": 411.9797668457031, + "fcm_dpo/q_t": 0.4265487790107727, + "grad_norm": 37.17644119262695, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.1041452884674072, + "logits/rejected": -1.132645845413208, + "logps/chosen": -873.485107421875, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1327.6015625, + "loss": 1.16, + "margin_dpo/margin_mean": 411.9798278808594, + "margin_dpo/margin_std": 713.0839233398438, + "step": 653 + }, + { + "KL/chosen_KL_mean": -741.3517456054688, + "KL/mean": -936.8746337890625, + "KL/rejected_KL_mean": -1132.3974609375, + "KL/std": 555.907470703125, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.0008047446608543396, + "fcm_dpo/delta": 0.08804425597190857, + "fcm_dpo/margin": 391.0456237792969, + "fcm_dpo/q_t": 0.42928260564804077, + "grad_norm": 37.38283920288086, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.0600917339324951, + "logits/rejected": -1.0766520500183105, + "logps/chosen": -803.0798950195312, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1231.171142578125, + "loss": 1.1612, + "margin_dpo/margin_mean": 391.04559326171875, + "margin_dpo/margin_std": 656.9273681640625, + "step": 654 + }, + { + "KL/chosen_KL_mean": -693.721435546875, + "KL/mean": -896.3341064453125, + "KL/rejected_KL_mean": -1098.9466552734375, + "KL/std": 567.3012084960938, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0008171196095645428, + "fcm_dpo/delta": 0.0711965262889862, + "fcm_dpo/margin": 405.225341796875, + "fcm_dpo/q_t": 0.4252209961414337, + "grad_norm": 26.34759521484375, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.0273557901382446, + "logits/rejected": -1.0711013078689575, + "logps/chosen": -743.2982177734375, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1197.238525390625, + "loss": 1.155, + "margin_dpo/margin_mean": 405.225341796875, + "margin_dpo/margin_std": 682.19287109375, + "step": 655 + }, + { + "KL/chosen_KL_mean": -740.5970458984375, + "KL/mean": -1081.199951171875, + "KL/rejected_KL_mean": -1421.802978515625, + "KL/std": 747.5555419921875, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0008040533866733313, + "fcm_dpo/delta": -0.15599049627780914, + "fcm_dpo/margin": 681.2058715820312, + "fcm_dpo/q_t": 0.37769079208374023, + "grad_norm": 97.19782257080078, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.9206315875053406, + "logits/rejected": -0.9935299158096313, + "logps/chosen": -793.146484375, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1535.4775390625, + "loss": 0.993, + "margin_dpo/margin_mean": 681.2058715820312, + "margin_dpo/margin_std": 753.6560668945312, + "step": 656 + }, + { + "KL/chosen_KL_mean": -699.4947509765625, + "KL/mean": -975.9888305664062, + "KL/rejected_KL_mean": -1252.48291015625, + "KL/std": 722.8453369140625, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0007893036818131804, + "fcm_dpo/delta": -0.038185544312000275, + "fcm_dpo/margin": 552.9881591796875, + "fcm_dpo/q_t": 0.4010157287120819, + "grad_norm": 42.80027389526367, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.066338300704956, + "logits/rejected": -1.1174534559249878, + "logps/chosen": -746.1952514648438, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1350.3978271484375, + "loss": 1.0763, + "margin_dpo/margin_mean": 552.9881591796875, + "margin_dpo/margin_std": 755.32861328125, + "step": 657 + }, + { + "KL/chosen_KL_mean": -765.38671875, + "KL/mean": -978.4747314453125, + "KL/rejected_KL_mean": -1191.5626220703125, + "KL/std": 554.6110229492188, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0007862721104174852, + "fcm_dpo/delta": -0.013587159104645252, + "fcm_dpo/margin": 426.17584228515625, + "fcm_dpo/q_t": 0.4214830994606018, + "grad_norm": 32.6586799621582, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -1.0501459836959839, + "logits/rejected": -1.0813417434692383, + "logps/chosen": -826.344970703125, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1287.5020751953125, + "loss": 1.1389, + "margin_dpo/margin_mean": 426.17584228515625, + "margin_dpo/margin_std": 629.2298583984375, + "step": 658 + }, + { + "KL/chosen_KL_mean": -670.141845703125, + "KL/mean": -880.6414794921875, + "KL/rejected_KL_mean": -1091.14111328125, + "KL/std": 566.8834228515625, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0007875190349295735, + "fcm_dpo/delta": 0.007923174649477005, + "fcm_dpo/margin": 420.999267578125, + "fcm_dpo/q_t": 0.4236389398574829, + "grad_norm": 42.868064880371094, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.9645726084709167, + "logits/rejected": -0.9426984786987305, + "logps/chosen": -746.8848876953125, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1178.612060546875, + "loss": 1.1383, + "margin_dpo/margin_mean": 420.999267578125, + "margin_dpo/margin_std": 621.8170166015625, + "step": 659 + }, + { + "KL/chosen_KL_mean": -714.7052612304688, + "KL/mean": -988.732177734375, + "KL/rejected_KL_mean": -1262.7591552734375, + "KL/std": 661.3277587890625, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0007852361304685473, + "fcm_dpo/delta": -0.03172639012336731, + "fcm_dpo/margin": 548.0538330078125, + "fcm_dpo/q_t": 0.4016149640083313, + "grad_norm": 40.47431945800781, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.071812629699707, + "logits/rejected": -1.0957788228988647, + "logps/chosen": -773.753173828125, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1338.71923828125, + "loss": 1.0735, + "margin_dpo/margin_mean": 548.0538330078125, + "margin_dpo/margin_std": 712.7533569335938, + "step": 660 + }, + { + "KL/chosen_KL_mean": -627.312744140625, + "KL/mean": -883.2666625976562, + "KL/rejected_KL_mean": -1139.220703125, + "KL/std": 739.2320556640625, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.0007851315313018858, + "fcm_dpo/delta": -0.002257194370031357, + "fcm_dpo/margin": 511.9078063964844, + "fcm_dpo/q_t": 0.41154104471206665, + "grad_norm": 55.59241485595703, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.0536426305770874, + "logits/rejected": -1.075774908065796, + "logps/chosen": -677.9867553710938, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1225.226318359375, + "loss": 1.0971, + "margin_dpo/margin_mean": 511.90777587890625, + "margin_dpo/margin_std": 752.6546630859375, + "step": 661 + }, + { + "KL/chosen_KL_mean": -734.9159545898438, + "KL/mean": -926.56005859375, + "KL/rejected_KL_mean": -1118.2041015625, + "KL/std": 594.729736328125, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0007894099107943475, + "fcm_dpo/delta": 0.045974310487508774, + "fcm_dpo/margin": 383.28826904296875, + "fcm_dpo/q_t": 0.4311929941177368, + "grad_norm": 39.52452850341797, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -0.9560129642486572, + "logits/rejected": -0.9562016725540161, + "logps/chosen": -804.177001953125, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1207.260009765625, + "loss": 1.1865, + "margin_dpo/margin_mean": 383.2882385253906, + "margin_dpo/margin_std": 741.8018798828125, + "step": 662 + }, + { + "KL/chosen_KL_mean": -703.7575073242188, + "KL/mean": -941.9609375, + "KL/rejected_KL_mean": -1180.164306640625, + "KL/std": 699.5870361328125, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0007910075364634395, + "fcm_dpo/delta": 0.02405317686498165, + "fcm_dpo/margin": 476.4067687988281, + "fcm_dpo/q_t": 0.41665565967559814, + "grad_norm": 28.90571403503418, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.0561808347702026, + "logits/rejected": -1.0852770805358887, + "logps/chosen": -768.636474609375, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1294.0897216796875, + "loss": 1.135, + "margin_dpo/margin_mean": 476.40679931640625, + "margin_dpo/margin_std": 811.1563720703125, + "step": 663 + }, + { + "KL/chosen_KL_mean": -743.4021606445312, + "KL/mean": -1011.9903564453125, + "KL/rejected_KL_mean": -1280.57861328125, + "KL/std": 690.6788940429688, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.000787506578490138, + "fcm_dpo/delta": -0.0243989210575819, + "fcm_dpo/margin": 537.1763916015625, + "fcm_dpo/q_t": 0.40481919050216675, + "grad_norm": 36.49754333496094, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.0182390213012695, + "logits/rejected": -1.0577185153961182, + "logps/chosen": -804.2906494140625, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1386.100341796875, + "loss": 1.085, + "margin_dpo/margin_mean": 537.1763916015625, + "margin_dpo/margin_std": 740.1445922851562, + "step": 664 + }, + { + "KL/chosen_KL_mean": -664.7102661132812, + "KL/mean": -887.0160522460938, + "KL/rejected_KL_mean": -1109.3218994140625, + "KL/std": 564.665771484375, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0007948251441121101, + "fcm_dpo/delta": 0.048331864178180695, + "fcm_dpo/margin": 444.61163330078125, + "fcm_dpo/q_t": 0.4180948734283447, + "grad_norm": 50.349945068359375, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.083099126815796, + "logits/rejected": -1.1118075847625732, + "logps/chosen": -725.2744140625, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1194.130615234375, + "loss": 1.1151, + "margin_dpo/margin_mean": 444.61163330078125, + "margin_dpo/margin_std": 581.520751953125, + "step": 665 + }, + { + "KL/chosen_KL_mean": -692.0473022460938, + "KL/mean": -917.10595703125, + "KL/rejected_KL_mean": -1142.16455078125, + "KL/std": 573.5545654296875, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0008010470774024725, + "fcm_dpo/delta": 0.04093039035797119, + "fcm_dpo/margin": 450.1172790527344, + "fcm_dpo/q_t": 0.4178573787212372, + "grad_norm": 37.140350341796875, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.9361131191253662, + "logits/rejected": -0.9491368532180786, + "logps/chosen": -756.46728515625, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1238.05615234375, + "loss": 1.1231, + "margin_dpo/margin_mean": 450.11724853515625, + "margin_dpo/margin_std": 647.04296875, + "step": 666 + }, + { + "KL/chosen_KL_mean": -808.5471801757812, + "KL/mean": -1070.89990234375, + "KL/rejected_KL_mean": -1333.2528076171875, + "KL/std": 635.421630859375, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0007992868777364492, + "fcm_dpo/delta": -0.020395569503307343, + "fcm_dpo/margin": 524.70556640625, + "fcm_dpo/q_t": 0.4043000340461731, + "grad_norm": 45.71607971191406, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -1.0048682689666748, + "logits/rejected": -1.010033369064331, + "logps/chosen": -877.82421875, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1421.0882568359375, + "loss": 1.0841, + "margin_dpo/margin_mean": 524.70556640625, + "margin_dpo/margin_std": 707.081298828125, + "step": 667 + }, + { + "KL/chosen_KL_mean": -856.620849609375, + "KL/mean": -1033.541748046875, + "KL/rejected_KL_mean": -1210.4625244140625, + "KL/std": 679.413330078125, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0008017306099645793, + "fcm_dpo/delta": 0.017811667174100876, + "fcm_dpo/margin": 353.84173583984375, + "fcm_dpo/q_t": 0.43770578503608704, + "grad_norm": 54.70947265625, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.04750394821167, + "logits/rejected": -1.0539287328720093, + "logps/chosen": -929.224853515625, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1314.20166015625, + "loss": 1.2241, + "margin_dpo/margin_mean": 353.84173583984375, + "margin_dpo/margin_std": 823.2427978515625, + "step": 668 + }, + { + "KL/chosen_KL_mean": -633.5390625, + "KL/mean": -899.0052490234375, + "KL/rejected_KL_mean": -1164.471435546875, + "KL/std": 626.609375, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0008012630278244615, + "fcm_dpo/delta": -0.026555201038718224, + "fcm_dpo/margin": 530.9324951171875, + "fcm_dpo/q_t": 0.40271323919296265, + "grad_norm": 25.43838882446289, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.0056214332580566, + "logits/rejected": -1.0185025930404663, + "logps/chosen": -679.655517578125, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1242.3958740234375, + "loss": 1.0801, + "margin_dpo/margin_mean": 530.9324951171875, + "margin_dpo/margin_std": 709.9252319335938, + "step": 669 + }, + { + "KL/chosen_KL_mean": -618.0423583984375, + "KL/mean": -875.5628662109375, + "KL/rejected_KL_mean": -1133.0833740234375, + "KL/std": 589.53759765625, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0008007909636944532, + "fcm_dpo/delta": -0.0133003368973732, + "fcm_dpo/margin": 515.041015625, + "fcm_dpo/q_t": 0.40536800026893616, + "grad_norm": 34.9423942565918, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.9251531958580017, + "logits/rejected": -0.9469876885414124, + "logps/chosen": -680.3881225585938, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1230.02392578125, + "loss": 1.0929, + "margin_dpo/margin_mean": 515.041015625, + "margin_dpo/margin_std": 706.8955078125, + "step": 670 + }, + { + "KL/chosen_KL_mean": -784.6771240234375, + "KL/mean": -990.4012451171875, + "KL/rejected_KL_mean": -1196.125244140625, + "KL/std": 604.6219482421875, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.0007969874422997236, + "fcm_dpo/delta": 0.002577702747657895, + "fcm_dpo/margin": 411.44818115234375, + "fcm_dpo/q_t": 0.4233209490776062, + "grad_norm": 32.40316390991211, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.0493581295013428, + "logits/rejected": -1.08070707321167, + "logps/chosen": -832.67724609375, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1279.944580078125, + "loss": 1.166, + "margin_dpo/margin_mean": 411.44818115234375, + "margin_dpo/margin_std": 733.5623779296875, + "step": 671 + }, + { + "KL/chosen_KL_mean": -856.775390625, + "KL/mean": -1065.29736328125, + "KL/rejected_KL_mean": -1273.8193359375, + "KL/std": 702.3109741210938, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0007994970656000078, + "fcm_dpo/delta": 0.014430626295506954, + "fcm_dpo/margin": 417.0439147949219, + "fcm_dpo/q_t": 0.42548415064811707, + "grad_norm": 54.434112548828125, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.102608561515808, + "logits/rejected": -1.1088229417800903, + "logps/chosen": -915.3587036132812, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1366.95947265625, + "loss": 1.1746, + "margin_dpo/margin_mean": 417.0439147949219, + "margin_dpo/margin_std": 789.5299072265625, + "step": 672 + }, + { + "KL/chosen_KL_mean": -780.2869873046875, + "KL/mean": -977.6544189453125, + "KL/rejected_KL_mean": -1175.021728515625, + "KL/std": 606.736572265625, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0008088820613920689, + "fcm_dpo/delta": 0.08342120796442032, + "fcm_dpo/margin": 394.73468017578125, + "fcm_dpo/q_t": 0.427710622549057, + "grad_norm": 31.43630027770996, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.057429313659668, + "logits/rejected": -1.0626882314682007, + "logps/chosen": -827.01025390625, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1260.31787109375, + "loss": 1.1618, + "margin_dpo/margin_mean": 394.73468017578125, + "margin_dpo/margin_std": 686.6727294921875, + "step": 673 + }, + { + "KL/chosen_KL_mean": -636.2152709960938, + "KL/mean": -899.39892578125, + "KL/rejected_KL_mean": -1162.58251953125, + "KL/std": 589.59228515625, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0008113991352729499, + "fcm_dpo/delta": -0.028346270322799683, + "fcm_dpo/margin": 526.367431640625, + "fcm_dpo/q_t": 0.40438514947891235, + "grad_norm": 44.58892059326172, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.9934746623039246, + "logits/rejected": -1.018500566482544, + "logps/chosen": -681.6607666015625, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1232.6285400390625, + "loss": 1.0783, + "margin_dpo/margin_mean": 526.367431640625, + "margin_dpo/margin_std": 706.4654541015625, + "step": 674 + }, + { + "KL/chosen_KL_mean": -721.628662109375, + "KL/mean": -985.6380615234375, + "KL/rejected_KL_mean": -1249.6474609375, + "KL/std": 652.9459228515625, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0007961175870150328, + "fcm_dpo/delta": -0.0762547105550766, + "fcm_dpo/margin": 528.0186767578125, + "fcm_dpo/q_t": 0.40585023164749146, + "grad_norm": 25.741762161254883, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.0328726768493652, + "logits/rejected": -1.0453965663909912, + "logps/chosen": -765.8049926757812, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1323.739501953125, + "loss": 1.086, + "margin_dpo/margin_mean": 528.0187377929688, + "margin_dpo/margin_std": 707.399169921875, + "step": 675 + }, + { + "KL/chosen_KL_mean": -769.4755859375, + "KL/mean": -1020.7625732421875, + "KL/rejected_KL_mean": -1272.0496826171875, + "KL/std": 625.9122924804688, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0007972304010763764, + "fcm_dpo/delta": -0.0007403884083032608, + "fcm_dpo/margin": 502.5740966796875, + "fcm_dpo/q_t": 0.40832406282424927, + "grad_norm": 23.287567138671875, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -0.9682983160018921, + "logits/rejected": -0.9879252910614014, + "logps/chosen": -840.8740844726562, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1360.408447265625, + "loss": 1.0877, + "margin_dpo/margin_mean": 502.5741271972656, + "margin_dpo/margin_std": 643.8828125, + "step": 676 + }, + { + "KL/chosen_KL_mean": -765.024658203125, + "KL/mean": -1007.5933837890625, + "KL/rejected_KL_mean": -1250.162109375, + "KL/std": 634.95361328125, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.0007969552534632385, + "fcm_dpo/delta": 0.013901378028094769, + "fcm_dpo/margin": 485.13739013671875, + "fcm_dpo/q_t": 0.4156782031059265, + "grad_norm": 37.60026931762695, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.05983304977417, + "logits/rejected": -1.0595300197601318, + "logps/chosen": -821.5521240234375, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1328.388671875, + "loss": 1.1185, + "margin_dpo/margin_mean": 485.13739013671875, + "margin_dpo/margin_std": 769.8028564453125, + "step": 677 + }, + { + "KL/chosen_KL_mean": -653.5613403320312, + "KL/mean": -944.23828125, + "KL/rejected_KL_mean": -1234.915283203125, + "KL/std": 704.4305419921875, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.000794127699919045, + "fcm_dpo/delta": -0.06477434188127518, + "fcm_dpo/margin": 581.3538818359375, + "fcm_dpo/q_t": 0.39638668298721313, + "grad_norm": 29.61897850036621, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -0.99812912940979, + "logits/rejected": -1.040347933769226, + "logps/chosen": -699.69580078125, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1315.519775390625, + "loss": 1.0684, + "margin_dpo/margin_mean": 581.3538818359375, + "margin_dpo/margin_std": 800.283447265625, + "step": 678 + }, + { + "KL/chosen_KL_mean": -732.1544189453125, + "KL/mean": -934.273193359375, + "KL/rejected_KL_mean": -1136.39208984375, + "KL/std": 565.7081298828125, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.0007927644182927907, + "fcm_dpo/delta": 0.030509470030665398, + "fcm_dpo/margin": 404.2376708984375, + "fcm_dpo/q_t": 0.42723962664604187, + "grad_norm": 34.52847671508789, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -1.0031286478042603, + "logits/rejected": -0.9956998229026794, + "logps/chosen": -782.4493408203125, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1212.990234375, + "loss": 1.1558, + "margin_dpo/margin_mean": 404.2376708984375, + "margin_dpo/margin_std": 665.67529296875, + "step": 679 + }, + { + "KL/chosen_KL_mean": -752.5553588867188, + "KL/mean": -1022.1414794921875, + "KL/rejected_KL_mean": -1291.7275390625, + "KL/std": 736.4444580078125, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.000785715994425118, + "fcm_dpo/delta": -0.025357451289892197, + "fcm_dpo/margin": 539.1722412109375, + "fcm_dpo/q_t": 0.40323930978775024, + "grad_norm": 54.28693771362305, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -0.9928523898124695, + "logits/rejected": -1.0251951217651367, + "logps/chosen": -829.4710693359375, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1404.1123046875, + "loss": 1.0986, + "margin_dpo/margin_mean": 539.1722412109375, + "margin_dpo/margin_std": 798.4219970703125, + "step": 680 + }, + { + "KL/chosen_KL_mean": -751.6201171875, + "KL/mean": -957.3773193359375, + "KL/rejected_KL_mean": -1163.1343994140625, + "KL/std": 610.318359375, + "epoch": 1.0, + "fcm_dpo/beta": 0.0007862101774662733, + "fcm_dpo/delta": -0.03235675394535065, + "fcm_dpo/margin": 411.5142822265625, + "fcm_dpo/q_t": 0.42578789591789246, + "grad_norm": 41.02858352661133, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.0574129819869995, + "logits/rejected": -1.0785119533538818, + "logps/chosen": -812.577392578125, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1251.6923828125, + "loss": 1.1624, + "margin_dpo/margin_mean": 411.5142822265625, + "margin_dpo/margin_std": 679.608642578125, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.1157032691609667, + "train_runtime": 1740.0089, + "train_samples_per_second": 25.056, + "train_steps_per_second": 0.391 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}