commit aef2411924440c971e4c6d76120b7d778cd0d65d Author: ModelHub XC Date: Mon May 18 22:13:03 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.05 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..6956a57 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-helpful-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.05 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-helpful-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.05 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-helpful-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-helpful-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..04c8b3d --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0870012780118095, + "train_runtime": 1735.4876, + "train_samples": 43598, + "train_samples_per_second": 25.121, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..1382dfc --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610f053f6c7ac32f0047e4f62312de284e0d53f24a750779dd2a86fd24bec5f1 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..e834d83 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc67caa0cb3ace6828094d677144fdc8d175c6a10a57ba6c59f2de7e837b11f +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..86373ed --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85195968154e62a90d9e160dc9959321c5ab661916005abb46c088fd3adc3dc5 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..df3c3d4 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8801f3ea546a44d3015584d26dcce6cd97c0598ef8df86e66c3b4a8adcf2e413 +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..7197e55 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96f089d4073ef1ab0fbbb23ccc38b68fde6b8071b87dc7478910c3e74d6d32ab +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..d767923 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2f6f1805465277c6307397b2031af4ee5423411a58eab397982cabc038499ac +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..68bbf47 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53dc2838c0d108cd2a19ee1d7a532e7590b73c9c66c2b6d40b2af4d74e198c22 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..04c8b3d --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 0.0, + "train_loss": 1.0870012780118095, + "train_runtime": 1735.4876, + "train_samples": 43598, + "train_samples_per_second": 25.121, + "train_steps_per_second": 0.392 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..76dd581 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15706 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 200, + "global_step": 681, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.00527191162109375, + "KL/mean": 0.016706019639968872, + "KL/rejected_KL_mean": 0.028141021728515625, + "KL/std": 0.272699236869812, + "epoch": 0.0014684287812041115, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.02287006378173828, + "fcm_dpo/q_t": 0.5005706548690796, + "grad_norm": 83.52085876464844, + "learning_rate": 0.0, + "logits/chosen": -0.4974287748336792, + "logits/rejected": -0.43299180269241333, + "logps/chosen": -50.1435661315918, + "logps/ref_chosen": -50.14883804321289, + "logps/ref_rejected": -74.1280517578125, + "logps/rejected": -74.09991455078125, + "loss": 1.389, + "margin_dpo/margin_mean": -0.02287048101425171, + "margin_dpo/margin_std": 0.41920793056488037, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.03498649597167969, + "KL/mean": -0.00212840735912323, + "KL/rejected_KL_mean": 0.030735015869140625, + "KL/std": 0.24797174334526062, + "epoch": 0.002936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06572261452674866, + "fcm_dpo/q_t": 0.5016425848007202, + "grad_norm": 72.19752502441406, + "learning_rate": 7.246376811594203e-09, + "logits/chosen": -0.49536412954330444, + "logits/rejected": -0.4594460427761078, + "logps/chosen": -52.65568923950195, + "logps/ref_chosen": -52.620704650878906, + "logps/ref_rejected": -75.30413818359375, + "logps/rejected": -75.27340698242188, + "loss": 1.3932, + "margin_dpo/margin_mean": -0.06572240591049194, + "margin_dpo/margin_std": 0.35048407316207886, + "step": 2 + }, + { + "KL/chosen_KL_mean": -0.017164230346679688, + "KL/mean": -0.008938148617744446, + "KL/rejected_KL_mean": -0.00070953369140625, + "KL/std": 0.25200146436691284, + "epoch": 0.004405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.016454964876174927, + "fcm_dpo/q_t": 0.500411331653595, + "grad_norm": 70.90198516845703, + "learning_rate": 1.4492753623188406e-08, + "logits/chosen": -0.4816335439682007, + "logits/rejected": -0.442240834236145, + "logps/chosen": -60.99876022338867, + "logps/ref_chosen": -60.981597900390625, + "logps/ref_rejected": -68.67259216308594, + "logps/rejected": -68.67330932617188, + "loss": 1.3883, + "margin_dpo/margin_mean": -0.016454339027404785, + "margin_dpo/margin_std": 0.3718428313732147, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.0011157989501953125, + "KL/mean": 0.030593425035476685, + "KL/rejected_KL_mean": 0.060070037841796875, + "KL/std": 0.2528386116027832, + "epoch": 0.005873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.058952271938323975, + "fcm_dpo/q_t": 0.5014750957489014, + "grad_norm": 72.30359649658203, + "learning_rate": 2.1739130434782606e-08, + "logits/chosen": -0.46882253885269165, + "logits/rejected": -0.44118714332580566, + "logps/chosen": -56.76659393310547, + "logps/ref_chosen": -56.7677116394043, + "logps/ref_rejected": -86.64710998535156, + "logps/rejected": -86.5870361328125, + "loss": 1.3926, + "margin_dpo/margin_mean": -0.058952778577804565, + "margin_dpo/margin_std": 0.3786373734474182, + "step": 4 + }, + { + "KL/chosen_KL_mean": 0.04130363464355469, + "KL/mean": 0.028823688626289368, + "KL/rejected_KL_mean": 0.016345977783203125, + "KL/std": 0.2790408730506897, + "epoch": 0.007342143906020558, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.024959057569503784, + "fcm_dpo/q_t": 0.4993762671947479, + "grad_norm": 89.41995239257812, + "learning_rate": 2.898550724637681e-08, + "logits/chosen": -0.5145682096481323, + "logits/rejected": -0.47082412242889404, + "logps/chosen": -53.81806945800781, + "logps/ref_chosen": -53.859375, + "logps/ref_rejected": -84.14918518066406, + "logps/rejected": -84.13284301757812, + "loss": 1.3842, + "margin_dpo/margin_mean": 0.024959444999694824, + "margin_dpo/margin_std": 0.38526445627212524, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.04564094543457031, + "KL/mean": -0.020483508706092834, + "KL/rejected_KL_mean": 0.00467681884765625, + "KL/std": 0.26991650462150574, + "epoch": 0.00881057268722467, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0503142774105072, + "fcm_dpo/q_t": 0.50125652551651, + "grad_norm": 92.1827621459961, + "learning_rate": 3.6231884057971014e-08, + "logits/chosen": -0.5298000574111938, + "logits/rejected": -0.4896334409713745, + "logps/chosen": -63.053123474121094, + "logps/ref_chosen": -63.007484436035156, + "logps/ref_rejected": -92.64534759521484, + "logps/rejected": -92.64067077636719, + "loss": 1.3918, + "margin_dpo/margin_mean": -0.05031454563140869, + "margin_dpo/margin_std": 0.4164780378341675, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.05684852600097656, + "KL/mean": 0.05007433891296387, + "KL/rejected_KL_mean": 0.043300628662109375, + "KL/std": 0.2920554578304291, + "epoch": 0.010279001468428781, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.013551175594329834, + "fcm_dpo/q_t": 0.4996613562107086, + "grad_norm": 82.23982238769531, + "learning_rate": 4.347826086956521e-08, + "logits/chosen": -0.5054950714111328, + "logits/rejected": -0.4716752767562866, + "logps/chosen": -57.71797180175781, + "logps/ref_chosen": -57.774818420410156, + "logps/ref_rejected": -103.92059326171875, + "logps/rejected": -103.87728881835938, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.013551592826843262, + "margin_dpo/margin_std": 0.43186938762664795, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.017724990844726562, + "KL/mean": 0.02704182267189026, + "KL/rejected_KL_mean": 0.0363616943359375, + "KL/std": 0.3007723093032837, + "epoch": 0.011747430249632892, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.018639206886291504, + "fcm_dpo/q_t": 0.500464916229248, + "grad_norm": 78.59706115722656, + "learning_rate": 5.0724637681159424e-08, + "logits/chosen": -0.5154826641082764, + "logits/rejected": -0.4903911352157593, + "logps/chosen": -58.69831085205078, + "logps/ref_chosen": -58.716033935546875, + "logps/ref_rejected": -79.3114242553711, + "logps/rejected": -79.27506256103516, + "loss": 1.3886, + "margin_dpo/margin_mean": -0.01863950490951538, + "margin_dpo/margin_std": 0.3895518183708191, + "step": 8 + }, + { + "KL/chosen_KL_mean": 0.00308990478515625, + "KL/mean": -0.0010503679513931274, + "KL/rejected_KL_mean": -0.00518798828125, + "KL/std": 0.28215163946151733, + "epoch": 0.013215859030837005, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.008274465799331665, + "fcm_dpo/q_t": 0.499793142080307, + "grad_norm": 85.33476257324219, + "learning_rate": 5.797101449275362e-08, + "logits/chosen": -0.5284153819084167, + "logits/rejected": -0.4838245213031769, + "logps/chosen": -69.86375427246094, + "logps/ref_chosen": -69.8668441772461, + "logps/ref_rejected": -99.6026611328125, + "logps/rejected": -99.60784912109375, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.008274048566818237, + "margin_dpo/margin_std": 0.4120550751686096, + "step": 9 + }, + { + "KL/chosen_KL_mean": 0.007932662963867188, + "KL/mean": 0.01091562956571579, + "KL/rejected_KL_mean": 0.0139007568359375, + "KL/std": 0.2518649697303772, + "epoch": 0.014684287812041116, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.00596599280834198, + "fcm_dpo/q_t": 0.5001487135887146, + "grad_norm": 70.95951080322266, + "learning_rate": 6.521739130434782e-08, + "logits/chosen": -0.4845648407936096, + "logits/rejected": -0.4408423602581024, + "logps/chosen": -48.349754333496094, + "logps/ref_chosen": -48.35768508911133, + "logps/ref_rejected": -80.37206268310547, + "logps/rejected": -80.35816192626953, + "loss": 1.3872, + "margin_dpo/margin_mean": -0.00596579909324646, + "margin_dpo/margin_std": 0.3506859540939331, + "step": 10 + }, + { + "KL/chosen_KL_mean": 0.06185722351074219, + "KL/mean": 0.0011502206325531006, + "KL/rejected_KL_mean": -0.0595550537109375, + "KL/std": 0.27010950446128845, + "epoch": 0.016152716593245228, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12140899896621704, + "fcm_dpo/q_t": 0.4969657063484192, + "grad_norm": 67.91629028320312, + "learning_rate": 7.246376811594203e-08, + "logits/chosen": -0.4723697900772095, + "logits/rejected": -0.44793397188186646, + "logps/chosen": -52.95499801635742, + "logps/ref_chosen": -53.01685333251953, + "logps/ref_rejected": -87.78038024902344, + "logps/rejected": -87.83993530273438, + "loss": 1.3745, + "margin_dpo/margin_mean": 0.12140867114067078, + "margin_dpo/margin_std": 0.3221217393875122, + "step": 11 + }, + { + "KL/chosen_KL_mean": 0.0077342987060546875, + "KL/mean": -0.026624195277690887, + "KL/rejected_KL_mean": -0.060977935791015625, + "KL/std": 0.26408517360687256, + "epoch": 0.01762114537444934, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06871287524700165, + "fcm_dpo/q_t": 0.49828261137008667, + "grad_norm": 95.73152923583984, + "learning_rate": 7.971014492753623e-08, + "logits/chosen": -0.5401338338851929, + "logits/rejected": -0.5040255784988403, + "logps/chosen": -61.79770278930664, + "logps/ref_chosen": -61.80543518066406, + "logps/ref_rejected": -104.8582763671875, + "logps/rejected": -104.91925048828125, + "loss": 1.3797, + "margin_dpo/margin_mean": 0.06871210038661957, + "margin_dpo/margin_std": 0.33830663561820984, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.0069599151611328125, + "KL/mean": -0.017511114478111267, + "KL/rejected_KL_mean": -0.04198455810546875, + "KL/std": 0.3143480122089386, + "epoch": 0.01908957415565345, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.048950523138046265, + "fcm_dpo/q_t": 0.4987792670726776, + "grad_norm": 79.25921630859375, + "learning_rate": 8.695652173913042e-08, + "logits/chosen": -0.4748424291610718, + "logits/rejected": -0.4463379681110382, + "logps/chosen": -64.2533950805664, + "logps/ref_chosen": -64.2603530883789, + "logps/ref_rejected": -87.20307922363281, + "logps/rejected": -87.24507141113281, + "loss": 1.382, + "margin_dpo/margin_mean": 0.048950642347335815, + "margin_dpo/margin_std": 0.44883352518081665, + "step": 13 + }, + { + "KL/chosen_KL_mean": -0.021654129028320312, + "KL/mean": 0.0024417340755462646, + "KL/rejected_KL_mean": 0.0265350341796875, + "KL/std": 0.2979321777820587, + "epoch": 0.020558002936857563, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.04818415641784668, + "fcm_dpo/q_t": 0.5012027025222778, + "grad_norm": 85.64781188964844, + "learning_rate": 9.420289855072464e-08, + "logits/chosen": -0.48348790407180786, + "logits/rejected": -0.4443298876285553, + "logps/chosen": -58.131866455078125, + "logps/ref_chosen": -58.11021041870117, + "logps/ref_rejected": -104.04708099365234, + "logps/rejected": -104.02054595947266, + "loss": 1.3916, + "margin_dpo/margin_mean": -0.048183709383010864, + "margin_dpo/margin_std": 0.44763094186782837, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.005901336669921875, + "KL/mean": -0.012522846460342407, + "KL/rejected_KL_mean": -0.019145965576171875, + "KL/std": 0.21723134815692902, + "epoch": 0.022026431718061675, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.013249039649963379, + "fcm_dpo/q_t": 0.49966850876808167, + "grad_norm": 64.4452133178711, + "learning_rate": 1.0144927536231885e-07, + "logits/chosen": -0.498607337474823, + "logits/rejected": -0.48026347160339355, + "logps/chosen": -56.972808837890625, + "logps/ref_chosen": -56.96691131591797, + "logps/ref_rejected": -80.80863952636719, + "logps/rejected": -80.82778930664062, + "loss": 1.3853, + "margin_dpo/margin_mean": 0.013249337673187256, + "margin_dpo/margin_std": 0.3437996804714203, + "step": 15 + }, + { + "KL/chosen_KL_mean": -0.027292251586914062, + "KL/mean": -0.009725570678710938, + "KL/rejected_KL_mean": 0.007839202880859375, + "KL/std": 0.2627600431442261, + "epoch": 0.023494860499265784, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.035129427909851074, + "fcm_dpo/q_t": 0.5008782148361206, + "grad_norm": 84.21294403076172, + "learning_rate": 1.0869565217391303e-07, + "logits/chosen": -0.5192116498947144, + "logits/rejected": -0.47780126333236694, + "logps/chosen": -61.76718521118164, + "logps/ref_chosen": -61.739891052246094, + "logps/ref_rejected": -84.36947631835938, + "logps/rejected": -84.36163330078125, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.035130053758621216, + "margin_dpo/margin_std": 0.35660019516944885, + "step": 16 + }, + { + "KL/chosen_KL_mean": 0.0523681640625, + "KL/mean": 0.011688470840454102, + "KL/rejected_KL_mean": -0.02899169921875, + "KL/std": 0.2670317590236664, + "epoch": 0.024963289280469897, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08135926723480225, + "fcm_dpo/q_t": 0.49796566367149353, + "grad_norm": 78.71238708496094, + "learning_rate": 1.1594202898550725e-07, + "logits/chosen": -0.48786813020706177, + "logits/rejected": -0.4491962194442749, + "logps/chosen": -67.65796661376953, + "logps/ref_chosen": -67.71033477783203, + "logps/ref_rejected": -85.37865447998047, + "logps/rejected": -85.40764617919922, + "loss": 1.3785, + "margin_dpo/margin_mean": 0.08135849237442017, + "margin_dpo/margin_std": 0.38243818283081055, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.039516448974609375, + "KL/mean": -0.002365991473197937, + "KL/rejected_KL_mean": -0.04425048828125, + "KL/std": 0.2152850478887558, + "epoch": 0.02643171806167401, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08376440405845642, + "fcm_dpo/q_t": 0.4979061186313629, + "grad_norm": 81.61917877197266, + "learning_rate": 1.2318840579710146e-07, + "logits/chosen": -0.493377149105072, + "logits/rejected": -0.43791770935058594, + "logps/chosen": -47.699974060058594, + "logps/ref_chosen": -47.7394905090332, + "logps/ref_rejected": -75.4722900390625, + "logps/rejected": -75.51654052734375, + "loss": 1.3782, + "margin_dpo/margin_mean": 0.08376404643058777, + "margin_dpo/margin_std": 0.2884919345378876, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.016864776611328125, + "KL/mean": -0.012162566184997559, + "KL/rejected_KL_mean": -0.041187286376953125, + "KL/std": 0.2685026526451111, + "epoch": 0.027900146842878122, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.058047711849212646, + "fcm_dpo/q_t": 0.4985480308532715, + "grad_norm": 73.1733627319336, + "learning_rate": 1.3043478260869563e-07, + "logits/chosen": -0.45881736278533936, + "logits/rejected": -0.40635204315185547, + "logps/chosen": -70.18849182128906, + "logps/ref_chosen": -70.20536041259766, + "logps/ref_rejected": -89.7575912475586, + "logps/rejected": -89.79878234863281, + "loss": 1.3808, + "margin_dpo/margin_mean": 0.05804792046546936, + "margin_dpo/margin_std": 0.3469369411468506, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.010059356689453125, + "KL/mean": -0.0335158109664917, + "KL/rejected_KL_mean": -0.07708358764648438, + "KL/std": 0.2843707203865051, + "epoch": 0.02936857562408223, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.08714669942855835, + "fcm_dpo/q_t": 0.4978218078613281, + "grad_norm": 74.1033935546875, + "learning_rate": 1.3768115942028986e-07, + "logits/chosen": -0.5568749904632568, + "logits/rejected": -0.5010716915130615, + "logps/chosen": -50.793182373046875, + "logps/ref_chosen": -50.80324172973633, + "logps/ref_rejected": -78.82334899902344, + "logps/rejected": -78.90043640136719, + "loss": 1.378, + "margin_dpo/margin_mean": 0.08714690804481506, + "margin_dpo/margin_std": 0.3879868984222412, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.018102645874023438, + "KL/mean": -0.054987095296382904, + "KL/rejected_KL_mean": -0.12807846069335938, + "KL/std": 0.2719336450099945, + "epoch": 0.030837004405286344, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1461833268404007, + "fcm_dpo/q_t": 0.49634718894958496, + "grad_norm": 77.20344543457031, + "learning_rate": 1.4492753623188405e-07, + "logits/chosen": -0.5189667344093323, + "logits/rejected": -0.49694347381591797, + "logps/chosen": -50.04491424560547, + "logps/ref_chosen": -50.063018798828125, + "logps/ref_rejected": -77.86878967285156, + "logps/rejected": -77.99687194824219, + "loss": 1.372, + "margin_dpo/margin_mean": 0.14618362486362457, + "margin_dpo/margin_std": 0.33233124017715454, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.026437759399414062, + "KL/mean": -0.06921970844268799, + "KL/rejected_KL_mean": -0.16487503051757812, + "KL/std": 0.2893703877925873, + "epoch": 0.032305433186490456, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1913151741027832, + "fcm_dpo/q_t": 0.49521979689598083, + "grad_norm": 84.33903503417969, + "learning_rate": 1.5217391304347825e-07, + "logits/chosen": -0.4624967873096466, + "logits/rejected": -0.41778409481048584, + "logps/chosen": -59.03119659423828, + "logps/ref_chosen": -59.05763626098633, + "logps/ref_rejected": -97.50466918945312, + "logps/rejected": -97.66954803466797, + "loss": 1.3677, + "margin_dpo/margin_mean": 0.1913146674633026, + "margin_dpo/margin_std": 0.4002724289894104, + "step": 22 + }, + { + "KL/chosen_KL_mean": 0.03466796875, + "KL/mean": -0.07599049806594849, + "KL/rejected_KL_mean": -0.1866474151611328, + "KL/std": 0.3174063563346863, + "epoch": 0.033773861967694566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.22131741046905518, + "fcm_dpo/q_t": 0.49447089433670044, + "grad_norm": 80.63067626953125, + "learning_rate": 1.5942028985507245e-07, + "logits/chosen": -0.5079987645149231, + "logits/rejected": -0.48651230335235596, + "logps/chosen": -60.04302978515625, + "logps/ref_chosen": -60.07769775390625, + "logps/ref_rejected": -81.13955688476562, + "logps/rejected": -81.32620239257812, + "loss": 1.3647, + "margin_dpo/margin_mean": 0.221317857503891, + "margin_dpo/margin_std": 0.40781861543655396, + "step": 23 + }, + { + "KL/chosen_KL_mean": 0.024261474609375, + "KL/mean": -0.0957612544298172, + "KL/rejected_KL_mean": -0.21578598022460938, + "KL/std": 0.27903175354003906, + "epoch": 0.03524229074889868, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24004700779914856, + "fcm_dpo/q_t": 0.4940027594566345, + "grad_norm": 82.3310546875, + "learning_rate": 1.6666666666666665e-07, + "logits/chosen": -0.5110853910446167, + "logits/rejected": -0.49455416202545166, + "logps/chosen": -44.26677322387695, + "logps/ref_chosen": -44.29103469848633, + "logps/ref_rejected": -99.12521362304688, + "logps/rejected": -99.34100341796875, + "loss": 1.3628, + "margin_dpo/margin_mean": 0.240046888589859, + "margin_dpo/margin_std": 0.3710036873817444, + "step": 24 + }, + { + "KL/chosen_KL_mean": 0.06036949157714844, + "KL/mean": -0.07619968056678772, + "KL/rejected_KL_mean": -0.2127685546875, + "KL/std": 0.35055142641067505, + "epoch": 0.03671071953010279, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2731361985206604, + "fcm_dpo/q_t": 0.49317657947540283, + "grad_norm": 73.42306518554688, + "learning_rate": 1.7391304347826085e-07, + "logits/chosen": -0.5052649974822998, + "logits/rejected": -0.4755502939224243, + "logps/chosen": -52.4766845703125, + "logps/ref_chosen": -52.537052154541016, + "logps/ref_rejected": -89.34219360351562, + "logps/rejected": -89.55496215820312, + "loss": 1.3597, + "margin_dpo/margin_mean": 0.2731362283229828, + "margin_dpo/margin_std": 0.45307743549346924, + "step": 25 + }, + { + "KL/chosen_KL_mean": 0.07877922058105469, + "KL/mean": -0.11963352560997009, + "KL/rejected_KL_mean": -0.31804656982421875, + "KL/std": 0.45149433612823486, + "epoch": 0.0381791483113069, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3968273997306824, + "fcm_dpo/q_t": 0.4900946319103241, + "grad_norm": 87.18598937988281, + "learning_rate": 1.8115942028985507e-07, + "logits/chosen": -0.5580284595489502, + "logits/rejected": -0.5269525051116943, + "logps/chosen": -53.84402847290039, + "logps/ref_chosen": -53.92280578613281, + "logps/ref_rejected": -103.35971069335938, + "logps/rejected": -103.67776489257812, + "loss": 1.3479, + "margin_dpo/margin_mean": 0.396827757358551, + "margin_dpo/margin_std": 0.5908505916595459, + "step": 26 + }, + { + "KL/chosen_KL_mean": 0.11952590942382812, + "KL/mean": -0.12472324073314667, + "KL/rejected_KL_mean": -0.3689727783203125, + "KL/std": 0.4575558304786682, + "epoch": 0.039647577092511016, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.4884994328022003, + "fcm_dpo/q_t": 0.4878042936325073, + "grad_norm": 93.66383361816406, + "learning_rate": 1.8840579710144927e-07, + "logits/chosen": -0.522944450378418, + "logits/rejected": -0.48664578795433044, + "logps/chosen": -42.77900314331055, + "logps/ref_chosen": -42.898529052734375, + "logps/ref_rejected": -98.72419738769531, + "logps/rejected": -99.09317016601562, + "loss": 1.3389, + "margin_dpo/margin_mean": 0.4884990155696869, + "margin_dpo/margin_std": 0.5788041353225708, + "step": 27 + }, + { + "KL/chosen_KL_mean": 0.0053863525390625, + "KL/mean": -0.1380460411310196, + "KL/rejected_KL_mean": -0.2814826965332031, + "KL/std": 0.418185830116272, + "epoch": 0.041116005873715125, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.28686580061912537, + "fcm_dpo/q_t": 0.49283337593078613, + "grad_norm": 75.52678680419922, + "learning_rate": 1.9565217391304347e-07, + "logits/chosen": -0.5053206086158752, + "logits/rejected": -0.44998180866241455, + "logps/chosen": -60.551116943359375, + "logps/ref_chosen": -60.55650329589844, + "logps/ref_rejected": -91.40111541748047, + "logps/rejected": -91.68260192871094, + "loss": 1.3587, + "margin_dpo/margin_mean": 0.2868659198284149, + "margin_dpo/margin_std": 0.5646681785583496, + "step": 28 + }, + { + "KL/chosen_KL_mean": 0.10842704772949219, + "KL/mean": -0.14363093674182892, + "KL/rejected_KL_mean": -0.3956871032714844, + "KL/std": 0.4754260182380676, + "epoch": 0.042584434654919234, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5041146278381348, + "fcm_dpo/q_t": 0.4874116778373718, + "grad_norm": 91.20955657958984, + "learning_rate": 2.028985507246377e-07, + "logits/chosen": -0.549689531326294, + "logits/rejected": -0.503177285194397, + "logps/chosen": -57.69935607910156, + "logps/ref_chosen": -57.80778503417969, + "logps/ref_rejected": -97.39434814453125, + "logps/rejected": -97.7900390625, + "loss": 1.3374, + "margin_dpo/margin_mean": 0.5041155815124512, + "margin_dpo/margin_std": 0.5955373644828796, + "step": 29 + }, + { + "KL/chosen_KL_mean": 0.14672279357910156, + "KL/mean": -0.1726699322462082, + "KL/rejected_KL_mean": -0.4920654296875, + "KL/std": 0.5893620848655701, + "epoch": 0.04405286343612335, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6387929916381836, + "fcm_dpo/q_t": 0.48405760526657104, + "grad_norm": 87.42900848388672, + "learning_rate": 2.1014492753623187e-07, + "logits/chosen": -0.47268378734588623, + "logits/rejected": -0.4410501718521118, + "logps/chosen": -52.43064880371094, + "logps/ref_chosen": -52.577369689941406, + "logps/ref_rejected": -98.48920440673828, + "logps/rejected": -98.98126983642578, + "loss": 1.3245, + "margin_dpo/margin_mean": 0.6387931108474731, + "margin_dpo/margin_std": 0.6522167921066284, + "step": 30 + }, + { + "KL/chosen_KL_mean": 0.12159347534179688, + "KL/mean": -0.13642428815364838, + "KL/rejected_KL_mean": -0.3944435119628906, + "KL/std": 0.5532899498939514, + "epoch": 0.04552129221732746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5160378217697144, + "fcm_dpo/q_t": 0.4871293008327484, + "grad_norm": 67.98377990722656, + "learning_rate": 2.1739130434782607e-07, + "logits/chosen": -0.503361165523529, + "logits/rejected": -0.45826011896133423, + "logps/chosen": -63.685325622558594, + "logps/ref_chosen": -63.806922912597656, + "logps/ref_rejected": -72.89400482177734, + "logps/rejected": -73.2884521484375, + "loss": 1.3366, + "margin_dpo/margin_mean": 0.5160377025604248, + "margin_dpo/margin_std": 0.6891189813613892, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.1765727996826172, + "KL/mean": -0.17987801134586334, + "KL/rejected_KL_mean": -0.5363311767578125, + "KL/std": 0.6821790933609009, + "epoch": 0.04698972099853157, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7129033803939819, + "fcm_dpo/q_t": 0.48223358392715454, + "grad_norm": 82.82377624511719, + "learning_rate": 2.2463768115942027e-07, + "logits/chosen": -0.49510425329208374, + "logits/rejected": -0.45262959599494934, + "logps/chosen": -62.562950134277344, + "logps/ref_chosen": -62.739524841308594, + "logps/ref_rejected": -89.3175048828125, + "logps/rejected": -89.85383605957031, + "loss": 1.3182, + "margin_dpo/margin_mean": 0.7129031419754028, + "margin_dpo/margin_std": 0.8785291910171509, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.12042808532714844, + "KL/mean": -0.18817251920700073, + "KL/rejected_KL_mean": -0.49677276611328125, + "KL/std": 0.5911757946014404, + "epoch": 0.048458149779735685, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6171959638595581, + "fcm_dpo/q_t": 0.4845910668373108, + "grad_norm": 72.13778686523438, + "learning_rate": 2.318840579710145e-07, + "logits/chosen": -0.48218345642089844, + "logits/rejected": -0.4558228552341461, + "logps/chosen": -53.14054489135742, + "logps/ref_chosen": -53.26097106933594, + "logps/ref_rejected": -87.8851318359375, + "logps/rejected": -88.38191223144531, + "loss": 1.3265, + "margin_dpo/margin_mean": 0.6171959638595581, + "margin_dpo/margin_std": 0.6186330914497375, + "step": 33 + }, + { + "KL/chosen_KL_mean": 0.06296157836914062, + "KL/mean": -0.34480684995651245, + "KL/rejected_KL_mean": -0.7525749206542969, + "KL/std": 0.7686243057250977, + "epoch": 0.049926578560939794, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.815531849861145, + "fcm_dpo/q_t": 0.4796821177005768, + "grad_norm": 77.3115005493164, + "learning_rate": 2.391304347826087e-07, + "logits/chosen": -0.49842262268066406, + "logits/rejected": -0.4809451103210449, + "logps/chosen": -50.754364013671875, + "logps/ref_chosen": -50.81732940673828, + "logps/ref_rejected": -101.92184448242188, + "logps/rejected": -102.67442321777344, + "loss": 1.3086, + "margin_dpo/margin_mean": 0.8155316114425659, + "margin_dpo/margin_std": 0.9295395016670227, + "step": 34 + }, + { + "KL/chosen_KL_mean": 0.12413406372070312, + "KL/mean": -0.4548630118370056, + "KL/rejected_KL_mean": -1.0338554382324219, + "KL/std": 0.9395226240158081, + "epoch": 0.0513950073421439, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.157991647720337, + "fcm_dpo/q_t": 0.4711979627609253, + "grad_norm": 83.07164001464844, + "learning_rate": 2.463768115942029e-07, + "logits/chosen": -0.4888343811035156, + "logits/rejected": -0.4522944688796997, + "logps/chosen": -50.90035629272461, + "logps/ref_chosen": -51.02449035644531, + "logps/ref_rejected": -106.82443237304688, + "logps/rejected": -107.85828399658203, + "loss": 1.2764, + "margin_dpo/margin_mean": 1.1579914093017578, + "margin_dpo/margin_std": 0.9997678995132446, + "step": 35 + }, + { + "KL/chosen_KL_mean": 0.04621315002441406, + "KL/mean": -0.5572574138641357, + "KL/rejected_KL_mean": -1.1607246398925781, + "KL/std": 1.1042115688323975, + "epoch": 0.05286343612334802, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.2069365978240967, + "fcm_dpo/q_t": 0.4699981212615967, + "grad_norm": 72.58307647705078, + "learning_rate": 2.536231884057971e-07, + "logits/chosen": -0.5532280206680298, + "logits/rejected": -0.5167073011398315, + "logps/chosen": -51.94527816772461, + "logps/ref_chosen": -51.991493225097656, + "logps/ref_rejected": -86.0406265258789, + "logps/rejected": -87.20135498046875, + "loss": 1.2733, + "margin_dpo/margin_mean": 1.206936240196228, + "margin_dpo/margin_std": 1.2257723808288574, + "step": 36 + }, + { + "KL/chosen_KL_mean": 0.00463104248046875, + "KL/mean": -0.4934205710887909, + "KL/rejected_KL_mean": -0.9914779663085938, + "KL/std": 1.0511497259140015, + "epoch": 0.05433186490455213, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.996109127998352, + "fcm_dpo/q_t": 0.4753045439720154, + "grad_norm": 61.719268798828125, + "learning_rate": 2.6086956521739126e-07, + "logits/chosen": -0.5050272941589355, + "logits/rejected": -0.4607655107975006, + "logps/chosen": -62.80247497558594, + "logps/ref_chosen": -62.807106018066406, + "logps/ref_rejected": -77.89507293701172, + "logps/rejected": -78.88655090332031, + "loss": 1.2941, + "margin_dpo/margin_mean": 0.9961087703704834, + "margin_dpo/margin_std": 1.374046802520752, + "step": 37 + }, + { + "KL/chosen_KL_mean": 0.1590423583984375, + "KL/mean": -0.5295313596725464, + "KL/rejected_KL_mean": -1.2181015014648438, + "KL/std": 1.374910593032837, + "epoch": 0.055800293685756244, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3771462440490723, + "fcm_dpo/q_t": 0.4660264253616333, + "grad_norm": 69.59135437011719, + "learning_rate": 2.681159420289855e-07, + "logits/chosen": -0.5226507186889648, + "logits/rejected": -0.4899941682815552, + "logps/chosen": -48.231475830078125, + "logps/ref_chosen": -48.39051818847656, + "logps/ref_rejected": -97.91244506835938, + "logps/rejected": -99.13053894042969, + "loss": 1.2606, + "margin_dpo/margin_mean": 1.3771460056304932, + "margin_dpo/margin_std": 1.6853830814361572, + "step": 38 + }, + { + "KL/chosen_KL_mean": 0.0872802734375, + "KL/mean": -0.7359923124313354, + "KL/rejected_KL_mean": -1.55926513671875, + "KL/std": 1.2973854541778564, + "epoch": 0.05726872246696035, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.6465413570404053, + "fcm_dpo/q_t": 0.4591583013534546, + "grad_norm": 74.43284606933594, + "learning_rate": 2.753623188405797e-07, + "logits/chosen": -0.5463048219680786, + "logits/rejected": -0.5056887269020081, + "logps/chosen": -50.66319274902344, + "logps/ref_chosen": -50.75047302246094, + "logps/ref_rejected": -78.56951141357422, + "logps/rejected": -80.12877655029297, + "loss": 1.2332, + "margin_dpo/margin_mean": 1.6465413570404053, + "margin_dpo/margin_std": 1.3756870031356812, + "step": 39 + }, + { + "KL/chosen_KL_mean": 0.2025127410888672, + "KL/mean": -0.5609214305877686, + "KL/rejected_KL_mean": -1.3243637084960938, + "KL/std": 1.3878769874572754, + "epoch": 0.05873715124816446, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5268778800964355, + "fcm_dpo/q_t": 0.4623277187347412, + "grad_norm": 60.651695251464844, + "learning_rate": 2.8260869565217386e-07, + "logits/chosen": -0.5206544399261475, + "logits/rejected": -0.4907876253128052, + "logps/chosen": -57.782554626464844, + "logps/ref_chosen": -57.985069274902344, + "logps/ref_rejected": -74.3000717163086, + "logps/rejected": -75.62443542480469, + "loss": 1.2468, + "margin_dpo/margin_mean": 1.5268781185150146, + "margin_dpo/margin_std": 1.691199779510498, + "step": 40 + }, + { + "KL/chosen_KL_mean": 0.06011772155761719, + "KL/mean": -0.9236688613891602, + "KL/rejected_KL_mean": -1.9074478149414062, + "KL/std": 1.8921372890472412, + "epoch": 0.06020558002936858, + "fcm_dpo/beta": 0.10048507153987885, + "fcm_dpo/delta": 0.09654659032821655, + "fcm_dpo/margin": 1.967568039894104, + "fcm_dpo/q_t": 0.45162275433540344, + "grad_norm": 67.96371459960938, + "learning_rate": 2.898550724637681e-07, + "logits/chosen": -0.5215292572975159, + "logits/rejected": -0.48398417234420776, + "logps/chosen": -62.63569641113281, + "logps/ref_chosen": -62.69581604003906, + "logps/ref_rejected": -97.02352905273438, + "logps/rejected": -98.93097686767578, + "loss": 1.2084, + "margin_dpo/margin_mean": 1.9675687551498413, + "margin_dpo/margin_std": 1.9688289165496826, + "step": 41 + }, + { + "KL/chosen_KL_mean": 0.2504749298095703, + "KL/mean": -1.0469075441360474, + "KL/rejected_KL_mean": -2.3442840576171875, + "KL/std": 2.180742025375366, + "epoch": 0.06167400881057269, + "fcm_dpo/beta": 0.10202906280755997, + "fcm_dpo/delta": 0.13704819977283478, + "fcm_dpo/margin": 2.5947656631469727, + "fcm_dpo/q_t": 0.4359305202960968, + "grad_norm": 80.4867935180664, + "learning_rate": 2.971014492753623e-07, + "logits/chosen": -0.5434771776199341, + "logits/rejected": -0.49688485264778137, + "logps/chosen": -58.71595001220703, + "logps/ref_chosen": -58.966426849365234, + "logps/ref_rejected": -109.90837097167969, + "logps/rejected": -112.25265502929688, + "loss": 1.1538, + "margin_dpo/margin_mean": 2.5947659015655518, + "margin_dpo/margin_std": 2.4078125953674316, + "step": 42 + }, + { + "KL/chosen_KL_mean": 0.5522232055664062, + "KL/mean": -0.6794946789741516, + "KL/rejected_KL_mean": -1.9112205505371094, + "KL/std": 1.821549654006958, + "epoch": 0.0631424375917768, + "fcm_dpo/beta": 0.10344166308641434, + "fcm_dpo/delta": 0.14711913466453552, + "fcm_dpo/margin": 2.463439464569092, + "fcm_dpo/q_t": 0.4379476308822632, + "grad_norm": 73.6222915649414, + "learning_rate": 3.043478260869565e-07, + "logits/chosen": -0.5423204898834229, + "logits/rejected": -0.517362117767334, + "logps/chosen": -53.60377502441406, + "logps/ref_chosen": -54.15599822998047, + "logps/ref_rejected": -96.48019409179688, + "logps/rejected": -98.39141082763672, + "loss": 1.1583, + "margin_dpo/margin_mean": 2.463439702987671, + "margin_dpo/margin_std": 2.007230758666992, + "step": 43 + }, + { + "KL/chosen_KL_mean": 0.2297210693359375, + "KL/mean": -1.195556402206421, + "KL/rejected_KL_mean": -2.6208343505859375, + "KL/std": 2.280576705932617, + "epoch": 0.06461086637298091, + "fcm_dpo/beta": 0.10464578866958618, + "fcm_dpo/delta": 0.10327555239200592, + "fcm_dpo/margin": 2.8505544662475586, + "fcm_dpo/q_t": 0.4275043308734894, + "grad_norm": 81.44078063964844, + "learning_rate": 3.115942028985507e-07, + "logits/chosen": -0.4537878632545471, + "logits/rejected": -0.43426036834716797, + "logps/chosen": -49.848777770996094, + "logps/ref_chosen": -50.07849884033203, + "logps/ref_rejected": -108.78376007080078, + "logps/rejected": -111.40459442138672, + "loss": 1.1233, + "margin_dpo/margin_mean": 2.8505539894104004, + "margin_dpo/margin_std": 2.22520112991333, + "step": 44 + }, + { + "KL/chosen_KL_mean": 0.09650421142578125, + "KL/mean": -1.0149602890014648, + "KL/rejected_KL_mean": -2.1264266967773438, + "KL/std": 2.0060698986053467, + "epoch": 0.06607929515418502, + "fcm_dpo/beta": 0.1063193678855896, + "fcm_dpo/delta": 0.1655273139476776, + "fcm_dpo/margin": 2.2229318618774414, + "fcm_dpo/q_t": 0.4429905414581299, + "grad_norm": 65.44628143310547, + "learning_rate": 3.188405797101449e-07, + "logits/chosen": -0.46523338556289673, + "logits/rejected": -0.4522145688533783, + "logps/chosen": -48.318424224853516, + "logps/ref_chosen": -48.4149284362793, + "logps/ref_rejected": -77.93643188476562, + "logps/rejected": -80.06285095214844, + "loss": 1.181, + "margin_dpo/margin_mean": 2.2229318618774414, + "margin_dpo/margin_std": 2.4767987728118896, + "step": 45 + }, + { + "KL/chosen_KL_mean": 0.18115234375, + "KL/mean": -1.2868940830230713, + "KL/rejected_KL_mean": -2.75494384765625, + "KL/std": 2.65130352973938, + "epoch": 0.06754772393538913, + "fcm_dpo/beta": 0.10749006271362305, + "fcm_dpo/delta": 0.08564689010381699, + "fcm_dpo/margin": 2.936098098754883, + "fcm_dpo/q_t": 0.4249163269996643, + "grad_norm": 71.94483184814453, + "learning_rate": 3.260869565217391e-07, + "logits/chosen": -0.5257543325424194, + "logits/rejected": -0.47459375858306885, + "logps/chosen": -55.818275451660156, + "logps/ref_chosen": -55.999427795410156, + "logps/ref_rejected": -95.652587890625, + "logps/rejected": -98.40753173828125, + "loss": 1.1239, + "margin_dpo/margin_mean": 2.9360976219177246, + "margin_dpo/margin_std": 3.2204670906066895, + "step": 46 + }, + { + "KL/chosen_KL_mean": 0.41971397399902344, + "KL/mean": -1.100320816040039, + "KL/rejected_KL_mean": -2.6203575134277344, + "KL/std": 2.6521072387695312, + "epoch": 0.06901615271659324, + "fcm_dpo/beta": 0.10835415124893188, + "fcm_dpo/delta": 0.07160530984401703, + "fcm_dpo/margin": 3.040071487426758, + "fcm_dpo/q_t": 0.42072784900665283, + "grad_norm": 68.87890625, + "learning_rate": 3.333333333333333e-07, + "logits/chosen": -0.5598181486129761, + "logits/rejected": -0.5062609910964966, + "logps/chosen": -57.50636291503906, + "logps/ref_chosen": -57.92607879638672, + "logps/ref_rejected": -94.67920684814453, + "logps/rejected": -97.299560546875, + "loss": 1.1053, + "margin_dpo/margin_mean": 3.040071487426758, + "margin_dpo/margin_std": 2.725712776184082, + "step": 47 + }, + { + "KL/chosen_KL_mean": 0.0533294677734375, + "KL/mean": -1.5111095905303955, + "KL/rejected_KL_mean": -3.075550079345703, + "KL/std": 2.7189888954162598, + "epoch": 0.07048458149779736, + "fcm_dpo/beta": 0.10873937606811523, + "fcm_dpo/delta": 0.06080545857548714, + "fcm_dpo/margin": 3.128880500793457, + "fcm_dpo/q_t": 0.41845259070396423, + "grad_norm": 75.4119873046875, + "learning_rate": 3.4057971014492755e-07, + "logits/chosen": -0.5846500396728516, + "logits/rejected": -0.5260331630706787, + "logps/chosen": -57.134742736816406, + "logps/ref_chosen": -57.188072204589844, + "logps/ref_rejected": -88.0166015625, + "logps/rejected": -91.09214782714844, + "loss": 1.1042, + "margin_dpo/margin_mean": 3.128880262374878, + "margin_dpo/margin_std": 2.981379508972168, + "step": 48 + }, + { + "KL/chosen_KL_mean": 0.31061363220214844, + "KL/mean": -1.6151647567749023, + "KL/rejected_KL_mean": -3.540943145751953, + "KL/std": 3.465115785598755, + "epoch": 0.07195301027900147, + "fcm_dpo/beta": 0.10909098386764526, + "fcm_dpo/delta": -0.020696185529232025, + "fcm_dpo/margin": 3.8515536785125732, + "fcm_dpo/q_t": 0.4022524952888489, + "grad_norm": 79.5746841430664, + "learning_rate": 3.478260869565217e-07, + "logits/chosen": -0.5361356139183044, + "logits/rejected": -0.4760180115699768, + "logps/chosen": -61.374656677246094, + "logps/ref_chosen": -61.685272216796875, + "logps/ref_rejected": -83.76747131347656, + "logps/rejected": -87.30841827392578, + "loss": 1.0546, + "margin_dpo/margin_mean": 3.8515536785125732, + "margin_dpo/margin_std": 3.974454164505005, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.15990066528320312, + "KL/mean": -2.221952438354492, + "KL/rejected_KL_mean": -4.284000396728516, + "KL/std": 3.5255508422851562, + "epoch": 0.07342143906020558, + "fcm_dpo/beta": 0.10840219259262085, + "fcm_dpo/delta": -0.048164162784814835, + "fcm_dpo/margin": 4.124100208282471, + "fcm_dpo/q_t": 0.395396888256073, + "grad_norm": 66.33158874511719, + "learning_rate": 3.5507246376811595e-07, + "logits/chosen": -0.5621310472488403, + "logits/rejected": -0.526812732219696, + "logps/chosen": -58.88404083251953, + "logps/ref_chosen": -58.72413635253906, + "logps/ref_rejected": -96.35814666748047, + "logps/rejected": -100.64215087890625, + "loss": 1.0337, + "margin_dpo/margin_mean": 4.1240997314453125, + "margin_dpo/margin_std": 4.0859832763671875, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2936878204345703, + "KL/mean": -2.299487590789795, + "KL/rejected_KL_mean": -4.305290222167969, + "KL/std": 4.18043327331543, + "epoch": 0.07488986784140969, + "fcm_dpo/beta": 0.10777164250612259, + "fcm_dpo/delta": -0.03339504823088646, + "fcm_dpo/margin": 4.011598587036133, + "fcm_dpo/q_t": 0.4028114676475525, + "grad_norm": 55.99860382080078, + "learning_rate": 3.6231884057971015e-07, + "logits/chosen": -0.5201721787452698, + "logits/rejected": -0.48702794313430786, + "logps/chosen": -61.6673583984375, + "logps/ref_chosen": -61.3736686706543, + "logps/ref_rejected": -76.00199890136719, + "logps/rejected": -80.30729675292969, + "loss": 1.0701, + "margin_dpo/margin_mean": 4.011598587036133, + "margin_dpo/margin_std": 5.134487152099609, + "step": 51 + }, + { + "KL/chosen_KL_mean": 0.3607959747314453, + "KL/mean": -2.7140278816223145, + "KL/rejected_KL_mean": -5.788852691650391, + "KL/std": 4.7607879638671875, + "epoch": 0.0763582966226138, + "fcm_dpo/beta": 0.10591427981853485, + "fcm_dpo/delta": -0.2597273588180542, + "fcm_dpo/margin": 6.149646759033203, + "fcm_dpo/q_t": 0.3517502546310425, + "grad_norm": 61.00212860107422, + "learning_rate": 3.695652173913043e-07, + "logits/chosen": -0.5540838241577148, + "logits/rejected": -0.4985507130622864, + "logps/chosen": -51.9765625, + "logps/ref_chosen": -52.33735656738281, + "logps/ref_rejected": -79.97391510009766, + "logps/rejected": -85.76277160644531, + "loss": 0.9009, + "margin_dpo/margin_mean": 6.149646759033203, + "margin_dpo/margin_std": 5.092857837677002, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.19704818725585938, + "KL/mean": -3.3417558670043945, + "KL/rejected_KL_mean": -6.486461639404297, + "KL/std": 5.458329677581787, + "epoch": 0.07782672540381791, + "fcm_dpo/beta": 0.10362114012241364, + "fcm_dpo/delta": -0.2598329186439514, + "fcm_dpo/margin": 6.289419174194336, + "fcm_dpo/q_t": 0.3608250916004181, + "grad_norm": 59.65618133544922, + "learning_rate": 3.7681159420289855e-07, + "logits/chosen": -0.6251329183578491, + "logits/rejected": -0.6039801836013794, + "logps/chosen": -53.51169967651367, + "logps/ref_chosen": -53.31465148925781, + "logps/ref_rejected": -91.78359985351562, + "logps/rejected": -98.27005767822266, + "loss": 0.9373, + "margin_dpo/margin_mean": 6.289420127868652, + "margin_dpo/margin_std": 6.279295921325684, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.4445972442626953, + "KL/mean": -3.137702703475952, + "KL/rejected_KL_mean": -5.830802917480469, + "KL/std": 5.121060848236084, + "epoch": 0.07929515418502203, + "fcm_dpo/beta": 0.10144082456827164, + "fcm_dpo/delta": -0.15038657188415527, + "fcm_dpo/margin": 5.386206150054932, + "fcm_dpo/q_t": 0.3753092288970947, + "grad_norm": 55.26798629760742, + "learning_rate": 3.8405797101449274e-07, + "logits/chosen": -0.5941368341445923, + "logits/rejected": -0.5407606363296509, + "logps/chosen": -51.13325881958008, + "logps/ref_chosen": -50.68865966796875, + "logps/ref_rejected": -91.71539306640625, + "logps/rejected": -97.54619598388672, + "loss": 0.9709, + "margin_dpo/margin_mean": 5.386206150054932, + "margin_dpo/margin_std": 5.081514835357666, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.927947998046875, + "KL/mean": -4.146857261657715, + "KL/rejected_KL_mean": -7.365764617919922, + "KL/std": 6.101950168609619, + "epoch": 0.08076358296622614, + "fcm_dpo/beta": 0.09932823479175568, + "fcm_dpo/delta": -0.24702517688274384, + "fcm_dpo/margin": 6.4378156661987305, + "fcm_dpo/q_t": 0.36172401905059814, + "grad_norm": 53.780818939208984, + "learning_rate": 3.9130434782608694e-07, + "logits/chosen": -0.6389350891113281, + "logits/rejected": -0.5758472084999084, + "logps/chosen": -63.543182373046875, + "logps/ref_chosen": -62.615234375, + "logps/ref_rejected": -88.99349975585938, + "logps/rejected": -96.35926818847656, + "loss": 0.9584, + "margin_dpo/margin_mean": 6.437814712524414, + "margin_dpo/margin_std": 7.308642387390137, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.7108230590820312, + "KL/mean": -3.80918550491333, + "KL/rejected_KL_mean": -6.907550811767578, + "KL/std": 6.075893402099609, + "epoch": 0.08223201174743025, + "fcm_dpo/beta": 0.09705003350973129, + "fcm_dpo/delta": -0.20751234889030457, + "fcm_dpo/margin": 6.196725845336914, + "fcm_dpo/q_t": 0.36958831548690796, + "grad_norm": 47.083831787109375, + "learning_rate": 3.9855072463768114e-07, + "logits/chosen": -0.630185604095459, + "logits/rejected": -0.5887142419815063, + "logps/chosen": -58.643550872802734, + "logps/ref_chosen": -57.9327278137207, + "logps/ref_rejected": -94.1744384765625, + "logps/rejected": -101.08198547363281, + "loss": 0.9834, + "margin_dpo/margin_mean": 6.196725845336914, + "margin_dpo/margin_std": 7.354310035705566, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.7268028259277344, + "KL/mean": -4.224173545837402, + "KL/rejected_KL_mean": -7.7215423583984375, + "KL/std": 6.010547161102295, + "epoch": 0.08370044052863436, + "fcm_dpo/beta": 0.09477832913398743, + "fcm_dpo/delta": -0.27153968811035156, + "fcm_dpo/margin": 6.994732856750488, + "fcm_dpo/q_t": 0.351573646068573, + "grad_norm": 52.91410827636719, + "learning_rate": 4.057971014492754e-07, + "logits/chosen": -0.5690896511077881, + "logits/rejected": -0.5405641794204712, + "logps/chosen": -71.22209167480469, + "logps/ref_chosen": -70.49528503417969, + "logps/ref_rejected": -95.56546020507812, + "logps/rejected": -103.28700256347656, + "loss": 0.918, + "margin_dpo/margin_mean": 6.994732856750488, + "margin_dpo/margin_std": 6.630904674530029, + "step": 57 + }, + { + "KL/chosen_KL_mean": -1.010385513305664, + "KL/mean": -4.842105865478516, + "KL/rejected_KL_mean": -8.673824310302734, + "KL/std": 6.785933494567871, + "epoch": 0.08516886930983847, + "fcm_dpo/beta": 0.09193111956119537, + "fcm_dpo/delta": -0.31510716676712036, + "fcm_dpo/margin": 7.663437366485596, + "fcm_dpo/q_t": 0.349496066570282, + "grad_norm": 55.937931060791016, + "learning_rate": 4.1304347826086954e-07, + "logits/chosen": -0.5902031660079956, + "logits/rejected": -0.513656735420227, + "logps/chosen": -63.14332580566406, + "logps/ref_chosen": -62.13294219970703, + "logps/ref_rejected": -84.61729431152344, + "logps/rejected": -93.29112243652344, + "loss": 0.9191, + "margin_dpo/margin_mean": 7.663437843322754, + "margin_dpo/margin_std": 8.12877368927002, + "step": 58 + }, + { + "KL/chosen_KL_mean": -1.4919471740722656, + "KL/mean": -5.709989070892334, + "KL/rejected_KL_mean": -9.928028106689453, + "KL/std": 7.444372177124023, + "epoch": 0.08663729809104258, + "fcm_dpo/beta": 0.08817608654499054, + "fcm_dpo/delta": -0.35956332087516785, + "fcm_dpo/margin": 8.436084747314453, + "fcm_dpo/q_t": 0.34159553050994873, + "grad_norm": 52.03582763671875, + "learning_rate": 4.2028985507246374e-07, + "logits/chosen": -0.6292190551757812, + "logits/rejected": -0.5891801118850708, + "logps/chosen": -53.42447280883789, + "logps/ref_chosen": -51.932525634765625, + "logps/ref_rejected": -88.88520050048828, + "logps/rejected": -98.813232421875, + "loss": 0.8944, + "margin_dpo/margin_mean": 8.43608570098877, + "margin_dpo/margin_std": 8.554616928100586, + "step": 59 + }, + { + "KL/chosen_KL_mean": -2.6756553649902344, + "KL/mean": -6.041238784790039, + "KL/rejected_KL_mean": -9.406822204589844, + "KL/std": 6.815237998962402, + "epoch": 0.0881057268722467, + "fcm_dpo/beta": 0.08637680113315582, + "fcm_dpo/delta": -0.18736253678798676, + "fcm_dpo/margin": 6.731164932250977, + "fcm_dpo/q_t": 0.36844515800476074, + "grad_norm": 57.49814987182617, + "learning_rate": 4.2753623188405794e-07, + "logits/chosen": -0.606688380241394, + "logits/rejected": -0.5464133024215698, + "logps/chosen": -63.61784362792969, + "logps/ref_chosen": -60.94218826293945, + "logps/ref_rejected": -85.39340209960938, + "logps/rejected": -94.80021667480469, + "loss": 0.9875, + "margin_dpo/margin_mean": 6.731164932250977, + "margin_dpo/margin_std": 7.683201313018799, + "step": 60 + }, + { + "KL/chosen_KL_mean": -1.5235176086425781, + "KL/mean": -5.651108264923096, + "KL/rejected_KL_mean": -9.778701782226562, + "KL/std": 9.094491004943848, + "epoch": 0.08957415565345081, + "fcm_dpo/beta": 0.08403058350086212, + "fcm_dpo/delta": -0.3050088882446289, + "fcm_dpo/margin": 8.255186080932617, + "fcm_dpo/q_t": 0.3618362545967102, + "grad_norm": 48.30548858642578, + "learning_rate": 4.3478260869565214e-07, + "logits/chosen": -0.6002434492111206, + "logits/rejected": -0.5660552382469177, + "logps/chosen": -62.15704345703125, + "logps/ref_chosen": -60.633522033691406, + "logps/ref_rejected": -89.85249328613281, + "logps/rejected": -99.63119506835938, + "loss": 0.9711, + "margin_dpo/margin_mean": 8.2551851272583, + "margin_dpo/margin_std": 11.305606842041016, + "step": 61 + }, + { + "KL/chosen_KL_mean": -1.6561450958251953, + "KL/mean": -4.769914150238037, + "KL/rejected_KL_mean": -7.8836822509765625, + "KL/std": 6.6822967529296875, + "epoch": 0.09104258443465492, + "fcm_dpo/beta": 0.08298887312412262, + "fcm_dpo/delta": -0.11985031515359879, + "fcm_dpo/margin": 6.227537155151367, + "fcm_dpo/q_t": 0.38729119300842285, + "grad_norm": 48.09984588623047, + "learning_rate": 4.420289855072464e-07, + "logits/chosen": -0.5964776873588562, + "logits/rejected": -0.5614856481552124, + "logps/chosen": -57.806915283203125, + "logps/ref_chosen": -56.15077209472656, + "logps/ref_rejected": -75.56619262695312, + "logps/rejected": -83.44987487792969, + "loss": 1.038, + "margin_dpo/margin_mean": 6.227537631988525, + "margin_dpo/margin_std": 8.184688568115234, + "step": 62 + }, + { + "KL/chosen_KL_mean": -2.681711196899414, + "KL/mean": -6.847625732421875, + "KL/rejected_KL_mean": -11.013534545898438, + "KL/std": 8.392871856689453, + "epoch": 0.09251101321585903, + "fcm_dpo/beta": 0.08056771755218506, + "fcm_dpo/delta": -0.2820628583431244, + "fcm_dpo/margin": 8.331822395324707, + "fcm_dpo/q_t": 0.35215944051742554, + "grad_norm": 49.44816970825195, + "learning_rate": 4.4927536231884053e-07, + "logits/chosen": -0.6123620271682739, + "logits/rejected": -0.5682877898216248, + "logps/chosen": -75.8291015625, + "logps/ref_chosen": -73.14739227294922, + "logps/ref_rejected": -97.61006164550781, + "logps/rejected": -108.62359619140625, + "loss": 0.9343, + "margin_dpo/margin_mean": 8.33182144165039, + "margin_dpo/margin_std": 8.752732276916504, + "step": 63 + }, + { + "KL/chosen_KL_mean": -1.037282943725586, + "KL/mean": -5.9826202392578125, + "KL/rejected_KL_mean": -10.927955627441406, + "KL/std": 9.150436401367188, + "epoch": 0.09397944199706314, + "fcm_dpo/beta": 0.07755880057811737, + "fcm_dpo/delta": -0.3850880563259125, + "fcm_dpo/margin": 9.890669822692871, + "fcm_dpo/q_t": 0.3408496081829071, + "grad_norm": 44.27583694458008, + "learning_rate": 4.5652173913043473e-07, + "logits/chosen": -0.605506181716919, + "logits/rejected": -0.5743293762207031, + "logps/chosen": -55.03588104248047, + "logps/ref_chosen": -53.998600006103516, + "logps/ref_rejected": -93.53019714355469, + "logps/rejected": -104.45816040039062, + "loss": 0.9102, + "margin_dpo/margin_mean": 9.890670776367188, + "margin_dpo/margin_std": 11.207742691040039, + "step": 64 + }, + { + "KL/chosen_KL_mean": -3.239490509033203, + "KL/mean": -8.281026840209961, + "KL/rejected_KL_mean": -13.322566986083984, + "KL/std": 10.2528076171875, + "epoch": 0.09544787077826726, + "fcm_dpo/beta": 0.07488435506820679, + "fcm_dpo/delta": -0.37056964635849, + "fcm_dpo/margin": 10.08306884765625, + "fcm_dpo/q_t": 0.3390854001045227, + "grad_norm": 44.772369384765625, + "learning_rate": 4.63768115942029e-07, + "logits/chosen": -0.6510547995567322, + "logits/rejected": -0.6380674839019775, + "logps/chosen": -68.07549285888672, + "logps/ref_chosen": -64.83599853515625, + "logps/ref_rejected": -109.94645690917969, + "logps/rejected": -123.26902770996094, + "loss": 0.9165, + "margin_dpo/margin_mean": 10.083067893981934, + "margin_dpo/margin_std": 11.16361141204834, + "step": 65 + }, + { + "KL/chosen_KL_mean": -3.0490779876708984, + "KL/mean": -7.599049091339111, + "KL/rejected_KL_mean": -12.149017333984375, + "KL/std": 9.199509620666504, + "epoch": 0.09691629955947137, + "fcm_dpo/beta": 0.07262101769447327, + "fcm_dpo/delta": -0.271577388048172, + "fcm_dpo/margin": 9.099935531616211, + "fcm_dpo/q_t": 0.36167746782302856, + "grad_norm": 41.545040130615234, + "learning_rate": 4.7101449275362313e-07, + "logits/chosen": -0.655698835849762, + "logits/rejected": -0.6236636638641357, + "logps/chosen": -54.49260330200195, + "logps/ref_chosen": -51.44352722167969, + "logps/ref_rejected": -75.63629913330078, + "logps/rejected": -87.78531646728516, + "loss": 0.9619, + "margin_dpo/margin_mean": 9.099935531616211, + "margin_dpo/margin_std": 11.122103691101074, + "step": 66 + }, + { + "KL/chosen_KL_mean": -2.6720447540283203, + "KL/mean": -7.3747406005859375, + "KL/rejected_KL_mean": -12.077438354492188, + "KL/std": 9.28006362915039, + "epoch": 0.09838472834067548, + "fcm_dpo/beta": 0.07105744630098343, + "fcm_dpo/delta": -0.27725303173065186, + "fcm_dpo/margin": 9.405391693115234, + "fcm_dpo/q_t": 0.3593307137489319, + "grad_norm": 41.94929885864258, + "learning_rate": 4.782608695652174e-07, + "logits/chosen": -0.6038053035736084, + "logits/rejected": -0.5614980459213257, + "logps/chosen": -62.012847900390625, + "logps/ref_chosen": -59.34080505371094, + "logps/ref_rejected": -72.78728485107422, + "logps/rejected": -84.8647232055664, + "loss": 0.9529, + "margin_dpo/margin_mean": 9.405391693115234, + "margin_dpo/margin_std": 11.189936637878418, + "step": 67 + }, + { + "KL/chosen_KL_mean": -3.055255889892578, + "KL/mean": -7.527899742126465, + "KL/rejected_KL_mean": -12.000545501708984, + "KL/std": 8.336349487304688, + "epoch": 0.09985315712187959, + "fcm_dpo/beta": 0.06936784088611603, + "fcm_dpo/delta": -0.22731636464595795, + "fcm_dpo/margin": 8.945289611816406, + "fcm_dpo/q_t": 0.36327165365219116, + "grad_norm": 40.98445129394531, + "learning_rate": 4.855072463768116e-07, + "logits/chosen": -0.6367079019546509, + "logits/rejected": -0.5765199661254883, + "logps/chosen": -68.26109313964844, + "logps/ref_chosen": -65.2058334350586, + "logps/ref_rejected": -77.20724487304688, + "logps/rejected": -89.20779418945312, + "loss": 0.9496, + "margin_dpo/margin_mean": 8.945289611816406, + "margin_dpo/margin_std": 9.330879211425781, + "step": 68 + }, + { + "KL/chosen_KL_mean": -3.5810699462890625, + "KL/mean": -9.05520248413086, + "KL/rejected_KL_mean": -14.529338836669922, + "KL/std": 9.87652587890625, + "epoch": 0.1013215859030837, + "fcm_dpo/beta": 0.06693008542060852, + "fcm_dpo/delta": -0.346579372882843, + "fcm_dpo/margin": 10.948261260986328, + "fcm_dpo/q_t": 0.3431288003921509, + "grad_norm": 42.85259246826172, + "learning_rate": 4.927536231884058e-07, + "logits/chosen": -0.6404001712799072, + "logits/rejected": -0.6165236830711365, + "logps/chosen": -63.40031051635742, + "logps/ref_chosen": -59.81924057006836, + "logps/ref_rejected": -103.38886260986328, + "logps/rejected": -117.91819763183594, + "loss": 0.8894, + "margin_dpo/margin_mean": 10.948262214660645, + "margin_dpo/margin_std": 10.901615142822266, + "step": 69 + }, + { + "KL/chosen_KL_mean": -5.2194671630859375, + "KL/mean": -11.088415145874023, + "KL/rejected_KL_mean": -16.957359313964844, + "KL/std": 12.024399757385254, + "epoch": 0.1027900146842878, + "fcm_dpo/beta": 0.0648043155670166, + "fcm_dpo/delta": -0.3743746280670166, + "fcm_dpo/margin": 11.73788833618164, + "fcm_dpo/q_t": 0.34193557500839233, + "grad_norm": 44.00492858886719, + "learning_rate": 5e-07, + "logits/chosen": -0.6287994384765625, + "logits/rejected": -0.5946310758590698, + "logps/chosen": -67.15010833740234, + "logps/ref_chosen": -61.930641174316406, + "logps/ref_rejected": -91.06078338623047, + "logps/rejected": -108.01814270019531, + "loss": 0.9036, + "margin_dpo/margin_mean": 11.737889289855957, + "margin_dpo/margin_std": 12.800237655639648, + "step": 70 + }, + { + "KL/chosen_KL_mean": -4.846014022827148, + "KL/mean": -11.838199615478516, + "KL/rejected_KL_mean": -18.830387115478516, + "KL/std": 12.62130355834961, + "epoch": 0.10425844346549193, + "fcm_dpo/beta": 0.061709873378276825, + "fcm_dpo/delta": -0.4846467971801758, + "fcm_dpo/margin": 13.984375, + "fcm_dpo/q_t": 0.32341229915618896, + "grad_norm": 39.75981903076172, + "learning_rate": 4.999967061337492e-07, + "logits/chosen": -0.6493800282478333, + "logits/rejected": -0.6095075607299805, + "logps/chosen": -66.59635162353516, + "logps/ref_chosen": -61.750335693359375, + "logps/ref_rejected": -97.33662414550781, + "logps/rejected": -116.16700744628906, + "loss": 0.837, + "margin_dpo/margin_mean": 13.984375953674316, + "margin_dpo/margin_std": 13.748950004577637, + "step": 71 + }, + { + "KL/chosen_KL_mean": -5.580968856811523, + "KL/mean": -12.921693801879883, + "KL/rejected_KL_mean": -20.262420654296875, + "KL/std": 13.505867004394531, + "epoch": 0.10572687224669604, + "fcm_dpo/beta": 0.05879486724734306, + "fcm_dpo/delta": -0.4848293662071228, + "fcm_dpo/margin": 14.681446075439453, + "fcm_dpo/q_t": 0.3229914903640747, + "grad_norm": 41.22248840332031, + "learning_rate": 4.999868246217933e-07, + "logits/chosen": -0.6457772254943848, + "logits/rejected": -0.6087461709976196, + "logps/chosen": -71.63438415527344, + "logps/ref_chosen": -66.05341339111328, + "logps/ref_rejected": -95.2869873046875, + "logps/rejected": -115.54940795898438, + "loss": 0.8629, + "margin_dpo/margin_mean": 14.681446075439453, + "margin_dpo/margin_std": 15.725846290588379, + "step": 72 + }, + { + "KL/chosen_KL_mean": -7.517843246459961, + "KL/mean": -14.800748825073242, + "KL/rejected_KL_mean": -22.083660125732422, + "KL/std": 17.561433792114258, + "epoch": 0.10719530102790015, + "fcm_dpo/beta": 0.05613835155963898, + "fcm_dpo/delta": -0.43695318698883057, + "fcm_dpo/margin": 14.565816879272461, + "fcm_dpo/q_t": 0.3509829044342041, + "grad_norm": 45.127349853515625, + "learning_rate": 4.999703557245192e-07, + "logits/chosen": -0.6935266256332397, + "logits/rejected": -0.653118908405304, + "logps/chosen": -73.77411651611328, + "logps/ref_chosen": -66.25627136230469, + "logps/ref_rejected": -90.45613098144531, + "logps/rejected": -112.539794921875, + "loss": 1.0029, + "margin_dpo/margin_mean": 14.565816879272461, + "margin_dpo/margin_std": 21.94605255126953, + "step": 73 + }, + { + "KL/chosen_KL_mean": -7.943721771240234, + "KL/mean": -15.891912460327148, + "KL/rejected_KL_mean": -23.840110778808594, + "KL/std": 19.308032989501953, + "epoch": 0.10866372980910426, + "fcm_dpo/beta": 0.05364230275154114, + "fcm_dpo/delta": -0.4738168716430664, + "fcm_dpo/margin": 15.896381378173828, + "fcm_dpo/q_t": 0.3405853509902954, + "grad_norm": 44.694664001464844, + "learning_rate": 4.999472998758977e-07, + "logits/chosen": -0.6524510383605957, + "logits/rejected": -0.6433727741241455, + "logps/chosen": -61.36860656738281, + "logps/ref_chosen": -53.42488098144531, + "logps/ref_rejected": -95.94693756103516, + "logps/rejected": -119.78704833984375, + "loss": 0.9488, + "margin_dpo/margin_mean": 15.896381378173828, + "margin_dpo/margin_std": 23.962190628051758, + "step": 74 + }, + { + "KL/chosen_KL_mean": -7.722936630249023, + "KL/mean": -18.847137451171875, + "KL/rejected_KL_mean": -29.971328735351562, + "KL/std": 19.825923919677734, + "epoch": 0.11013215859030837, + "fcm_dpo/beta": 0.050399381667375565, + "fcm_dpo/delta": -0.761603832244873, + "fcm_dpo/margin": 22.24839973449707, + "fcm_dpo/q_t": 0.2916664481163025, + "grad_norm": 36.36934280395508, + "learning_rate": 4.999176576834721e-07, + "logits/chosen": -0.6783813238143921, + "logits/rejected": -0.6700829267501831, + "logps/chosen": -59.584598541259766, + "logps/ref_chosen": -51.861663818359375, + "logps/ref_rejected": -111.25398254394531, + "logps/rejected": -141.22531127929688, + "loss": 0.7699, + "margin_dpo/margin_mean": 22.248397827148438, + "margin_dpo/margin_std": 22.400943756103516, + "step": 75 + }, + { + "KL/chosen_KL_mean": -9.420347213745117, + "KL/mean": -16.42698097229004, + "KL/rejected_KL_mean": -23.433605194091797, + "KL/std": 15.946723937988281, + "epoch": 0.11160058737151249, + "fcm_dpo/beta": 0.04817916452884674, + "fcm_dpo/delta": -0.2846575379371643, + "fcm_dpo/margin": 14.013269424438477, + "fcm_dpo/q_t": 0.35380053520202637, + "grad_norm": 39.79032516479492, + "learning_rate": 4.998814299283415e-07, + "logits/chosen": -0.7044748067855835, + "logits/rejected": -0.6629197597503662, + "logps/chosen": -62.68638610839844, + "logps/ref_chosen": -53.26603698730469, + "logps/ref_rejected": -78.21662902832031, + "logps/rejected": -101.65023803710938, + "loss": 0.9645, + "margin_dpo/margin_mean": 14.013269424438477, + "margin_dpo/margin_std": 17.342012405395508, + "step": 76 + }, + { + "KL/chosen_KL_mean": -8.207674026489258, + "KL/mean": -19.680038452148438, + "KL/rejected_KL_mean": -31.15240478515625, + "KL/std": 21.73928451538086, + "epoch": 0.1130690161527166, + "fcm_dpo/beta": 0.04527886211872101, + "fcm_dpo/delta": -0.677450954914093, + "fcm_dpo/margin": 22.944732666015625, + "fcm_dpo/q_t": 0.29555851221084595, + "grad_norm": 41.42151641845703, + "learning_rate": 4.998386175651409e-07, + "logits/chosen": -0.6860731244087219, + "logits/rejected": -0.647629976272583, + "logps/chosen": -66.30435180664062, + "logps/ref_chosen": -58.0966796875, + "logps/ref_rejected": -93.77361297607422, + "logps/rejected": -124.92601776123047, + "loss": 0.8147, + "margin_dpo/margin_mean": 22.944730758666992, + "margin_dpo/margin_std": 23.827333450317383, + "step": 77 + }, + { + "KL/chosen_KL_mean": -8.151559829711914, + "KL/mean": -16.818445205688477, + "KL/rejected_KL_mean": -25.485332489013672, + "KL/std": 17.356788635253906, + "epoch": 0.1145374449339207, + "fcm_dpo/beta": 0.043553952127695084, + "fcm_dpo/delta": -0.368335485458374, + "fcm_dpo/margin": 17.33376693725586, + "fcm_dpo/q_t": 0.34413814544677734, + "grad_norm": 36.57780838012695, + "learning_rate": 4.997892217220159e-07, + "logits/chosen": -0.6453582048416138, + "logits/rejected": -0.6197341680526733, + "logps/chosen": -63.765342712402344, + "logps/ref_chosen": -55.61378479003906, + "logps/ref_rejected": -84.93436431884766, + "logps/rejected": -110.41969299316406, + "loss": 0.9188, + "margin_dpo/margin_mean": 17.33376693725586, + "margin_dpo/margin_std": 19.793365478515625, + "step": 78 + }, + { + "KL/chosen_KL_mean": -8.366096496582031, + "KL/mean": -17.763093948364258, + "KL/rejected_KL_mean": -27.16008758544922, + "KL/std": 19.61853790283203, + "epoch": 0.11600587371512482, + "fcm_dpo/beta": 0.04178548604249954, + "fcm_dpo/delta": -0.4011922776699066, + "fcm_dpo/margin": 18.793994903564453, + "fcm_dpo/q_t": 0.34827283024787903, + "grad_norm": 32.12797164916992, + "learning_rate": 4.997332437005931e-07, + "logits/chosen": -0.6654260158538818, + "logits/rejected": -0.6358423829078674, + "logps/chosen": -63.81658172607422, + "logps/ref_chosen": -55.45048522949219, + "logps/ref_rejected": -87.64756774902344, + "logps/rejected": -114.80764770507812, + "loss": 0.9432, + "margin_dpo/margin_mean": 18.793994903564453, + "margin_dpo/margin_std": 24.024066925048828, + "step": 79 + }, + { + "KL/chosen_KL_mean": -11.082340240478516, + "KL/mean": -19.782291412353516, + "KL/rejected_KL_mean": -28.48224639892578, + "KL/std": 20.413394927978516, + "epoch": 0.11747430249632893, + "fcm_dpo/beta": 0.0402105450630188, + "fcm_dpo/delta": -0.31318235397338867, + "fcm_dpo/margin": 17.399906158447266, + "fcm_dpo/q_t": 0.3642631471157074, + "grad_norm": 35.137020111083984, + "learning_rate": 4.996706849759452e-07, + "logits/chosen": -0.7078701853752136, + "logits/rejected": -0.6634222269058228, + "logps/chosen": -69.60163116455078, + "logps/ref_chosen": -58.519290924072266, + "logps/ref_rejected": -87.54750061035156, + "logps/rejected": -116.02974700927734, + "loss": 0.9904, + "margin_dpo/margin_mean": 17.399904251098633, + "margin_dpo/margin_std": 23.675479888916016, + "step": 80 + }, + { + "KL/chosen_KL_mean": -9.967578887939453, + "KL/mean": -21.815593719482422, + "KL/rejected_KL_mean": -33.663612365722656, + "KL/std": 24.03949737548828, + "epoch": 0.11894273127753303, + "fcm_dpo/beta": 0.03838124871253967, + "fcm_dpo/delta": -0.5364028811454773, + "fcm_dpo/margin": 23.69603729248047, + "fcm_dpo/q_t": 0.3254055976867676, + "grad_norm": 35.88831329345703, + "learning_rate": 4.996015471965529e-07, + "logits/chosen": -0.7091889381408691, + "logits/rejected": -0.6791235208511353, + "logps/chosen": -76.41644287109375, + "logps/ref_chosen": -66.44886779785156, + "logps/ref_rejected": -129.66270446777344, + "logps/rejected": -163.32632446289062, + "loss": 0.8792, + "margin_dpo/margin_mean": 23.69603729248047, + "margin_dpo/margin_std": 28.50186538696289, + "step": 81 + }, + { + "KL/chosen_KL_mean": -12.100605010986328, + "KL/mean": -21.5577392578125, + "KL/rejected_KL_mean": -31.014873504638672, + "KL/std": 21.757911682128906, + "epoch": 0.12041116005873716, + "fcm_dpo/beta": 0.03735063970088959, + "fcm_dpo/delta": -0.3172946870326996, + "fcm_dpo/margin": 18.914270401000977, + "fcm_dpo/q_t": 0.3623591363430023, + "grad_norm": 41.56498336791992, + "learning_rate": 4.995258321842611e-07, + "logits/chosen": -0.6205165386199951, + "logits/rejected": -0.6074250340461731, + "logps/chosen": -64.33299255371094, + "logps/ref_chosen": -52.232383728027344, + "logps/ref_rejected": -90.74325561523438, + "logps/rejected": -121.75812530517578, + "loss": 1.044, + "margin_dpo/margin_mean": 18.914270401000977, + "margin_dpo/margin_std": 29.809494018554688, + "step": 82 + }, + { + "KL/chosen_KL_mean": -12.361867904663086, + "KL/mean": -23.320655822753906, + "KL/rejected_KL_mean": -34.27945327758789, + "KL/std": 22.589454650878906, + "epoch": 0.12187958883994127, + "fcm_dpo/beta": 0.035669997334480286, + "fcm_dpo/delta": -0.39825981855392456, + "fcm_dpo/margin": 21.917579650878906, + "fcm_dpo/q_t": 0.3431558609008789, + "grad_norm": 36.22183609008789, + "learning_rate": 4.994435419342304e-07, + "logits/chosen": -0.6784165501594543, + "logits/rejected": -0.6415207982063293, + "logps/chosen": -68.18925476074219, + "logps/ref_chosen": -55.82738494873047, + "logps/ref_rejected": -103.71589660644531, + "logps/rejected": -137.995361328125, + "loss": 0.9297, + "margin_dpo/margin_mean": 21.917579650878906, + "margin_dpo/margin_std": 26.579288482666016, + "step": 83 + }, + { + "KL/chosen_KL_mean": -10.984846115112305, + "KL/mean": -20.518203735351562, + "KL/rejected_KL_mean": -30.051559448242188, + "KL/std": 19.22481346130371, + "epoch": 0.12334801762114538, + "fcm_dpo/beta": 0.034514445811510086, + "fcm_dpo/delta": -0.268974244594574, + "fcm_dpo/margin": 19.066707611083984, + "fcm_dpo/q_t": 0.35393911600112915, + "grad_norm": 31.12311553955078, + "learning_rate": 4.993546786148857e-07, + "logits/chosen": -0.6781740188598633, + "logits/rejected": -0.6409305334091187, + "logps/chosen": -78.1610107421875, + "logps/ref_chosen": -67.1761703491211, + "logps/ref_rejected": -87.29859924316406, + "logps/rejected": -117.35015869140625, + "loss": 0.9372, + "margin_dpo/margin_mean": 19.066707611083984, + "margin_dpo/margin_std": 19.633758544921875, + "step": 84 + }, + { + "KL/chosen_KL_mean": -11.446046829223633, + "KL/mean": -21.150569915771484, + "KL/rejected_KL_mean": -30.855098724365234, + "KL/std": 19.64529037475586, + "epoch": 0.12481644640234948, + "fcm_dpo/beta": 0.03391923010349274, + "fcm_dpo/delta": -0.2667713761329651, + "fcm_dpo/margin": 19.409046173095703, + "fcm_dpo/q_t": 0.36208292841911316, + "grad_norm": 31.290390014648438, + "learning_rate": 4.992592445678582e-07, + "logits/chosen": -0.6304664611816406, + "logits/rejected": -0.5972647070884705, + "logps/chosen": -69.8526611328125, + "logps/ref_chosen": -58.4066162109375, + "logps/ref_rejected": -78.63880157470703, + "logps/rejected": -109.493896484375, + "loss": 0.9736, + "margin_dpo/margin_mean": 19.409046173095703, + "margin_dpo/margin_std": 23.33486557006836, + "step": 85 + }, + { + "KL/chosen_KL_mean": -14.669717788696289, + "KL/mean": -24.960693359375, + "KL/rejected_KL_mean": -35.25166702270508, + "KL/std": 25.916988372802734, + "epoch": 0.1262848751835536, + "fcm_dpo/beta": 0.032901618629693985, + "fcm_dpo/delta": -0.2864453196525574, + "fcm_dpo/margin": 20.581951141357422, + "fcm_dpo/q_t": 0.37629234790802, + "grad_norm": 39.49543380737305, + "learning_rate": 4.991572423079235e-07, + "logits/chosen": -0.7171605825424194, + "logits/rejected": -0.7046980857849121, + "logps/chosen": -70.80718231201172, + "logps/ref_chosen": -56.13746643066406, + "logps/ref_rejected": -88.12165069580078, + "logps/rejected": -123.37332153320312, + "loss": 1.0691, + "margin_dpo/margin_mean": 20.581951141357422, + "margin_dpo/margin_std": 35.439178466796875, + "step": 86 + }, + { + "KL/chosen_KL_mean": -14.323598861694336, + "KL/mean": -26.19344711303711, + "KL/rejected_KL_mean": -38.063297271728516, + "KL/std": 25.82670021057129, + "epoch": 0.1277533039647577, + "fcm_dpo/beta": 0.03160586953163147, + "fcm_dpo/delta": -0.36560332775115967, + "fcm_dpo/margin": 23.739696502685547, + "fcm_dpo/q_t": 0.34734636545181274, + "grad_norm": 31.173131942749023, + "learning_rate": 4.990486745229364e-07, + "logits/chosen": -0.7083604335784912, + "logits/rejected": -0.6805447340011597, + "logps/chosen": -69.9596939086914, + "logps/ref_chosen": -55.63609313964844, + "logps/ref_rejected": -95.46757507324219, + "logps/rejected": -133.5308837890625, + "loss": 0.9671, + "margin_dpo/margin_mean": 23.739696502685547, + "margin_dpo/margin_std": 31.024333953857422, + "step": 87 + }, + { + "KL/chosen_KL_mean": -18.130157470703125, + "KL/mean": -27.014564514160156, + "KL/rejected_KL_mean": -35.89897155761719, + "KL/std": 26.533187866210938, + "epoch": 0.12922173274596183, + "fcm_dpo/beta": 0.030946895480155945, + "fcm_dpo/delta": -0.15513765811920166, + "fcm_dpo/margin": 17.768814086914062, + "fcm_dpo/q_t": 0.3872056007385254, + "grad_norm": 33.197364807128906, + "learning_rate": 4.989335440737586e-07, + "logits/chosen": -0.658212423324585, + "logits/rejected": -0.6492829322814941, + "logps/chosen": -91.80130767822266, + "logps/ref_chosen": -73.67115020751953, + "logps/ref_rejected": -106.70849609375, + "logps/rejected": -142.6074676513672, + "loss": 1.0846, + "margin_dpo/margin_mean": 17.768814086914062, + "margin_dpo/margin_std": 27.587907791137695, + "step": 88 + }, + { + "KL/chosen_KL_mean": -10.722139358520508, + "KL/mean": -20.540828704833984, + "KL/rejected_KL_mean": -30.359527587890625, + "KL/std": 22.09980010986328, + "epoch": 0.13069016152716592, + "fcm_dpo/beta": 0.03054092824459076, + "fcm_dpo/delta": -0.2056586593389511, + "fcm_dpo/margin": 19.63738250732422, + "fcm_dpo/q_t": 0.3710666000843048, + "grad_norm": 28.453542709350586, + "learning_rate": 4.988118539941847e-07, + "logits/chosen": -0.7179343104362488, + "logits/rejected": -0.6833662986755371, + "logps/chosen": -71.34705352783203, + "logps/ref_chosen": -60.624916076660156, + "logps/ref_rejected": -82.08354949951172, + "logps/rejected": -112.44307708740234, + "loss": 0.9914, + "margin_dpo/margin_mean": 19.63738250732422, + "margin_dpo/margin_std": 24.839431762695312, + "step": 89 + }, + { + "KL/chosen_KL_mean": -14.105724334716797, + "KL/mean": -27.13547706604004, + "KL/rejected_KL_mean": -40.16523361206055, + "KL/std": 29.942901611328125, + "epoch": 0.13215859030837004, + "fcm_dpo/beta": 0.029609836637973785, + "fcm_dpo/delta": -0.38564369082450867, + "fcm_dpo/margin": 26.059507369995117, + "fcm_dpo/q_t": 0.3560681939125061, + "grad_norm": 30.310239791870117, + "learning_rate": 4.986836074908615e-07, + "logits/chosen": -0.6555777192115784, + "logits/rejected": -0.6643766164779663, + "logps/chosen": -67.39103698730469, + "logps/ref_chosen": -53.285308837890625, + "logps/ref_rejected": -111.54470825195312, + "logps/rejected": -151.70994567871094, + "loss": 0.9857, + "margin_dpo/margin_mean": 26.05950927734375, + "margin_dpo/margin_std": 36.949066162109375, + "step": 90 + }, + { + "KL/chosen_KL_mean": -14.317174911499023, + "KL/mean": -25.541458129882812, + "KL/rejected_KL_mean": -36.76573944091797, + "KL/std": 24.45753288269043, + "epoch": 0.13362701908957417, + "fcm_dpo/beta": 0.028681177645921707, + "fcm_dpo/delta": -0.2515973150730133, + "fcm_dpo/margin": 22.448570251464844, + "fcm_dpo/q_t": 0.3691549599170685, + "grad_norm": 29.94892692565918, + "learning_rate": 4.985488079432037e-07, + "logits/chosen": -0.6794449090957642, + "logits/rejected": -0.643684983253479, + "logps/chosen": -76.12013244628906, + "logps/ref_chosen": -61.802955627441406, + "logps/ref_rejected": -87.87395477294922, + "logps/rejected": -124.63969421386719, + "loss": 1.0094, + "margin_dpo/margin_mean": 22.448570251464844, + "margin_dpo/margin_std": 31.180084228515625, + "step": 91 + }, + { + "KL/chosen_KL_mean": -12.907651901245117, + "KL/mean": -24.19264030456543, + "KL/rejected_KL_mean": -35.477630615234375, + "KL/std": 25.02846908569336, + "epoch": 0.13509544787077826, + "fcm_dpo/beta": 0.028016380965709686, + "fcm_dpo/delta": -0.23956279456615448, + "fcm_dpo/margin": 22.56998062133789, + "fcm_dpo/q_t": 0.3695027232170105, + "grad_norm": 27.0372371673584, + "learning_rate": 4.984074589033043e-07, + "logits/chosen": -0.7169160842895508, + "logits/rejected": -0.693396270275116, + "logps/chosen": -64.54841613769531, + "logps/ref_chosen": -51.640769958496094, + "logps/ref_rejected": -77.88117980957031, + "logps/rejected": -113.35881042480469, + "loss": 1.0017, + "margin_dpo/margin_mean": 22.56998062133789, + "margin_dpo/margin_std": 30.188648223876953, + "step": 92 + }, + { + "KL/chosen_KL_mean": -14.480842590332031, + "KL/mean": -25.734779357910156, + "KL/rejected_KL_mean": -36.98870849609375, + "KL/std": 23.70608901977539, + "epoch": 0.13656387665198239, + "fcm_dpo/beta": 0.027309108525514603, + "fcm_dpo/delta": -0.22143405675888062, + "fcm_dpo/margin": 22.507863998413086, + "fcm_dpo/q_t": 0.3679991662502289, + "grad_norm": 27.40256118774414, + "learning_rate": 4.982595640958425e-07, + "logits/chosen": -0.738601803779602, + "logits/rejected": -0.6867475509643555, + "logps/chosen": -67.01008605957031, + "logps/ref_chosen": -52.529239654541016, + "logps/ref_rejected": -77.16075134277344, + "logps/rejected": -114.14945983886719, + "loss": 0.9724, + "margin_dpo/margin_mean": 22.507863998413086, + "margin_dpo/margin_std": 26.947500228881836, + "step": 93 + }, + { + "KL/chosen_KL_mean": -15.58909797668457, + "KL/mean": -28.327407836914062, + "KL/rejected_KL_mean": -41.06570816040039, + "KL/std": 26.550617218017578, + "epoch": 0.13803230543318648, + "fcm_dpo/beta": 0.026492692530155182, + "fcm_dpo/delta": -0.28588011860847473, + "fcm_dpo/margin": 25.476613998413086, + "fcm_dpo/q_t": 0.35494130849838257, + "grad_norm": 27.439083099365234, + "learning_rate": 4.98105127417984e-07, + "logits/chosen": -0.6954045295715332, + "logits/rejected": -0.6782183647155762, + "logps/chosen": -76.81170654296875, + "logps/ref_chosen": -61.22261047363281, + "logps/ref_rejected": -99.59902954101562, + "logps/rejected": -140.66473388671875, + "loss": 0.9351, + "margin_dpo/margin_mean": 25.476612091064453, + "margin_dpo/margin_std": 27.425670623779297, + "step": 94 + }, + { + "KL/chosen_KL_mean": -14.52149772644043, + "KL/mean": -24.788345336914062, + "KL/rejected_KL_mean": -35.055198669433594, + "KL/std": 24.6375732421875, + "epoch": 0.1395007342143906, + "fcm_dpo/beta": 0.026066306978464127, + "fcm_dpo/delta": -0.13932469487190247, + "fcm_dpo/margin": 20.533700942993164, + "fcm_dpo/q_t": 0.3804051876068115, + "grad_norm": 26.025171279907227, + "learning_rate": 4.979441529392784e-07, + "logits/chosen": -0.7163432836532593, + "logits/rejected": -0.68684983253479, + "logps/chosen": -67.0451431274414, + "logps/ref_chosen": -52.523643493652344, + "logps/ref_rejected": -75.8803482055664, + "logps/rejected": -110.935546875, + "loss": 1.0224, + "margin_dpo/margin_mean": 20.533700942993164, + "margin_dpo/margin_std": 25.788597106933594, + "step": 95 + }, + { + "KL/chosen_KL_mean": -13.763811111450195, + "KL/mean": -27.650653839111328, + "KL/rejected_KL_mean": -41.537498474121094, + "KL/std": 28.800777435302734, + "epoch": 0.14096916299559473, + "fcm_dpo/beta": 0.025319525972008705, + "fcm_dpo/delta": -0.31601524353027344, + "fcm_dpo/margin": 27.773691177368164, + "fcm_dpo/q_t": 0.35219550132751465, + "grad_norm": 26.12571144104004, + "learning_rate": 4.977766449015534e-07, + "logits/chosen": -0.7276456356048584, + "logits/rejected": -0.6974596977233887, + "logps/chosen": -75.92078399658203, + "logps/ref_chosen": -62.15697479248047, + "logps/ref_rejected": -96.59601593017578, + "logps/rejected": -138.13351440429688, + "loss": 0.9254, + "margin_dpo/margin_mean": 27.773693084716797, + "margin_dpo/margin_std": 31.26250457763672, + "step": 96 + }, + { + "KL/chosen_KL_mean": -15.030601501464844, + "KL/mean": -25.972335815429688, + "KL/rejected_KL_mean": -36.91407012939453, + "KL/std": 23.70777702331543, + "epoch": 0.14243759177679882, + "fcm_dpo/beta": 0.025046605616807938, + "fcm_dpo/delta": -0.15248806774616241, + "fcm_dpo/margin": 21.88347625732422, + "fcm_dpo/q_t": 0.37621963024139404, + "grad_norm": 27.10018539428711, + "learning_rate": 4.976026077188012e-07, + "logits/chosen": -0.6382083892822266, + "logits/rejected": -0.5908419489860535, + "logps/chosen": -69.67697143554688, + "logps/ref_chosen": -54.646366119384766, + "logps/ref_rejected": -76.96475219726562, + "logps/rejected": -113.87882232666016, + "loss": 0.9984, + "margin_dpo/margin_mean": 21.88347625732422, + "margin_dpo/margin_std": 23.601760864257812, + "step": 97 + }, + { + "KL/chosen_KL_mean": -18.270347595214844, + "KL/mean": -30.684240341186523, + "KL/rejected_KL_mean": -43.09813690185547, + "KL/std": 26.8064022064209, + "epoch": 0.14390602055800295, + "fcm_dpo/beta": 0.02441706880927086, + "fcm_dpo/delta": -0.21288006007671356, + "fcm_dpo/margin": 24.82779312133789, + "fcm_dpo/q_t": 0.36630043387413025, + "grad_norm": 27.9980525970459, + "learning_rate": 4.974220459770639e-07, + "logits/chosen": -0.6928224563598633, + "logits/rejected": -0.6749119758605957, + "logps/chosen": -83.52897644042969, + "logps/ref_chosen": -65.25862884521484, + "logps/ref_rejected": -96.5274887084961, + "logps/rejected": -139.62562561035156, + "loss": 1.0038, + "margin_dpo/margin_mean": 24.82779312133789, + "margin_dpo/margin_std": 31.736007690429688, + "step": 98 + }, + { + "KL/chosen_KL_mean": -14.74056625366211, + "KL/mean": -29.577295303344727, + "KL/rejected_KL_mean": -44.41402053833008, + "KL/std": 29.210227966308594, + "epoch": 0.14537444933920704, + "fcm_dpo/beta": 0.0237110685557127, + "fcm_dpo/delta": -0.31514137983322144, + "fcm_dpo/margin": 29.67345428466797, + "fcm_dpo/q_t": 0.3556155562400818, + "grad_norm": 24.59937858581543, + "learning_rate": 4.972349644343108e-07, + "logits/chosen": -0.657272458076477, + "logits/rejected": -0.6558362245559692, + "logps/chosen": -60.379051208496094, + "logps/ref_chosen": -45.638484954833984, + "logps/ref_rejected": -86.43793487548828, + "logps/rejected": -130.85195922851562, + "loss": 0.9386, + "margin_dpo/margin_mean": 29.6734561920166, + "margin_dpo/margin_std": 35.876800537109375, + "step": 99 + }, + { + "KL/chosen_KL_mean": -16.435449600219727, + "KL/mean": -25.48604393005371, + "KL/rejected_KL_mean": -34.53663635253906, + "KL/std": 23.471981048583984, + "epoch": 0.14684287812041116, + "fcm_dpo/beta": 0.02355112135410309, + "fcm_dpo/delta": -0.026980634778738022, + "fcm_dpo/margin": 18.101184844970703, + "fcm_dpo/q_t": 0.40622931718826294, + "grad_norm": 27.36819839477539, + "learning_rate": 4.970413680203148e-07, + "logits/chosen": -0.6757396459579468, + "logits/rejected": -0.6324243545532227, + "logps/chosen": -74.02943420410156, + "logps/ref_chosen": -57.59397888183594, + "logps/ref_rejected": -74.06021118164062, + "logps/rejected": -108.59684753417969, + "loss": 1.1247, + "margin_dpo/margin_mean": 18.101184844970703, + "margin_dpo/margin_std": 30.202056884765625, + "step": 100 + }, + { + "KL/chosen_KL_mean": -20.00939178466797, + "KL/mean": -30.913528442382812, + "KL/rejected_KL_mean": -41.817665100097656, + "KL/std": 28.39974021911621, + "epoch": 0.14831130690161526, + "fcm_dpo/beta": 0.02322225831449032, + "fcm_dpo/delta": -0.109965018928051, + "fcm_dpo/margin": 21.80826759338379, + "fcm_dpo/q_t": 0.39490771293640137, + "grad_norm": 27.217756271362305, + "learning_rate": 4.968412618365215e-07, + "logits/chosen": -0.6777423620223999, + "logits/rejected": -0.6437240242958069, + "logps/chosen": -81.65824890136719, + "logps/ref_chosen": -61.64885330200195, + "logps/ref_rejected": -83.18968200683594, + "logps/rejected": -125.00735473632812, + "loss": 1.0806, + "margin_dpo/margin_mean": 21.808269500732422, + "margin_dpo/margin_std": 34.3305778503418, + "step": 101 + }, + { + "KL/chosen_KL_mean": -22.16390037536621, + "KL/mean": -30.27094268798828, + "KL/rejected_KL_mean": -38.377986907958984, + "KL/std": 27.00689697265625, + "epoch": 0.14977973568281938, + "fcm_dpo/beta": 0.023358814418315887, + "fcm_dpo/delta": 0.020977407693862915, + "fcm_dpo/margin": 16.214086532592773, + "fcm_dpo/q_t": 0.4182248115539551, + "grad_norm": 30.8178653717041, + "learning_rate": 4.966346511559149e-07, + "logits/chosen": -0.7105182409286499, + "logits/rejected": -0.6647392511367798, + "logps/chosen": -86.24278259277344, + "logps/ref_chosen": -64.0788803100586, + "logps/ref_rejected": -68.18707275390625, + "logps/rejected": -106.5650634765625, + "loss": 1.1713, + "margin_dpo/margin_mean": 16.214086532592773, + "margin_dpo/margin_std": 31.576683044433594, + "step": 102 + }, + { + "KL/chosen_KL_mean": -16.63315200805664, + "KL/mean": -32.39291763305664, + "KL/rejected_KL_mean": -48.15269088745117, + "KL/std": 29.840614318847656, + "epoch": 0.1512481644640235, + "fcm_dpo/beta": 0.02269843965768814, + "fcm_dpo/delta": -0.32748711109161377, + "fcm_dpo/margin": 31.519527435302734, + "fcm_dpo/q_t": 0.34971487522125244, + "grad_norm": 25.520038604736328, + "learning_rate": 4.964215414228785e-07, + "logits/chosen": -0.6867813467979431, + "logits/rejected": -0.6520103216171265, + "logps/chosen": -77.93243408203125, + "logps/ref_chosen": -61.299278259277344, + "logps/ref_rejected": -93.57270812988281, + "logps/rejected": -141.72540283203125, + "loss": 0.9196, + "margin_dpo/margin_mean": 31.5195255279541, + "margin_dpo/margin_std": 35.207923889160156, + "step": 103 + }, + { + "KL/chosen_KL_mean": -18.6593074798584, + "KL/mean": -33.599281311035156, + "KL/rejected_KL_mean": -48.53925323486328, + "KL/std": 33.02933120727539, + "epoch": 0.1527165932452276, + "fcm_dpo/beta": 0.022114867344498634, + "fcm_dpo/delta": -0.2693977355957031, + "fcm_dpo/margin": 29.879947662353516, + "fcm_dpo/q_t": 0.36667945981025696, + "grad_norm": 25.26581382751465, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": -0.7141636610031128, + "logits/rejected": -0.6804061532020569, + "logps/chosen": -73.0320816040039, + "logps/ref_chosen": -54.372772216796875, + "logps/ref_rejected": -89.5647201538086, + "logps/rejected": -138.10397338867188, + "loss": 0.988, + "margin_dpo/margin_mean": 29.879947662353516, + "margin_dpo/margin_std": 39.69654846191406, + "step": 104 + }, + { + "KL/chosen_KL_mean": -17.245506286621094, + "KL/mean": -38.19187545776367, + "KL/rejected_KL_mean": -59.13824462890625, + "KL/std": 32.81090545654297, + "epoch": 0.15418502202643172, + "fcm_dpo/beta": 0.021134644746780396, + "fcm_dpo/delta": -0.5075461268424988, + "fcm_dpo/margin": 41.89273452758789, + "fcm_dpo/q_t": 0.31017786264419556, + "grad_norm": 25.08208656311035, + "learning_rate": 4.959758474331832e-07, + "logits/chosen": -0.7256340980529785, + "logits/rejected": -0.7023687958717346, + "logps/chosen": -71.88445281982422, + "logps/ref_chosen": -54.638946533203125, + "logps/ref_rejected": -97.97351837158203, + "logps/rejected": -157.11175537109375, + "loss": 0.7999, + "margin_dpo/margin_mean": 41.89273452758789, + "margin_dpo/margin_std": 35.73217010498047, + "step": 105 + }, + { + "KL/chosen_KL_mean": -19.42215919494629, + "KL/mean": -32.675418853759766, + "KL/rejected_KL_mean": -45.928688049316406, + "KL/std": 28.108577728271484, + "epoch": 0.15565345080763582, + "fcm_dpo/beta": 0.020594779402017593, + "fcm_dpo/delta": -0.15016362071037292, + "fcm_dpo/margin": 26.506515502929688, + "fcm_dpo/q_t": 0.37773293256759644, + "grad_norm": 24.609127044677734, + "learning_rate": 4.957432749209755e-07, + "logits/chosen": -0.662516713142395, + "logits/rejected": -0.6267033815383911, + "logps/chosen": -74.25505065917969, + "logps/ref_chosen": -54.83289337158203, + "logps/ref_rejected": -85.22461700439453, + "logps/rejected": -131.15330505371094, + "loss": 1.0027, + "margin_dpo/margin_mean": 26.506515502929688, + "margin_dpo/margin_std": 30.649978637695312, + "step": 106 + }, + { + "KL/chosen_KL_mean": -23.350513458251953, + "KL/mean": -38.28211975097656, + "KL/rejected_KL_mean": -53.213722229003906, + "KL/std": 33.904205322265625, + "epoch": 0.15712187958883994, + "fcm_dpo/beta": 0.02017119899392128, + "fcm_dpo/delta": -0.20935019850730896, + "fcm_dpo/margin": 29.863208770751953, + "fcm_dpo/q_t": 0.367611289024353, + "grad_norm": 24.215755462646484, + "learning_rate": 4.955042268449307e-07, + "logits/chosen": -0.685881495475769, + "logits/rejected": -0.6361589431762695, + "logps/chosen": -93.0583267211914, + "logps/ref_chosen": -69.70780944824219, + "logps/ref_rejected": -94.73950958251953, + "logps/rejected": -147.95323181152344, + "loss": 0.9882, + "margin_dpo/margin_mean": 29.86320686340332, + "margin_dpo/margin_std": 35.658416748046875, + "step": 107 + }, + { + "KL/chosen_KL_mean": -21.30466651916504, + "KL/mean": -38.65107727050781, + "KL/rejected_KL_mean": -55.99748992919922, + "KL/std": 37.09898376464844, + "epoch": 0.15859030837004406, + "fcm_dpo/beta": 0.01967495307326317, + "fcm_dpo/delta": -0.2925887405872345, + "fcm_dpo/margin": 34.69282531738281, + "fcm_dpo/q_t": 0.36451563239097595, + "grad_norm": 24.30539894104004, + "learning_rate": 4.952587095041881e-07, + "logits/chosen": -0.6991877555847168, + "logits/rejected": -0.6731090545654297, + "logps/chosen": -77.3145523071289, + "logps/ref_chosen": -56.0098876953125, + "logps/ref_rejected": -95.79601287841797, + "logps/rejected": -151.7935028076172, + "loss": 0.9954, + "margin_dpo/margin_mean": 34.69282531738281, + "margin_dpo/margin_std": 47.79713439941406, + "step": 108 + }, + { + "KL/chosen_KL_mean": -19.659822463989258, + "KL/mean": -38.451622009277344, + "KL/rejected_KL_mean": -57.24341583251953, + "KL/std": 36.00904846191406, + "epoch": 0.16005873715124816, + "fcm_dpo/beta": 0.019026821479201317, + "fcm_dpo/delta": -0.32746434211730957, + "fcm_dpo/margin": 37.583595275878906, + "fcm_dpo/q_t": 0.34695935249328613, + "grad_norm": 24.934322357177734, + "learning_rate": 4.95006729368358e-07, + "logits/chosen": -0.6155581474304199, + "logits/rejected": -0.5928876996040344, + "logps/chosen": -82.5453109741211, + "logps/ref_chosen": -62.88549041748047, + "logps/ref_rejected": -98.68573760986328, + "logps/rejected": -155.9291534423828, + "loss": 0.923, + "margin_dpo/margin_mean": 37.58359909057617, + "margin_dpo/margin_std": 40.89533996582031, + "step": 109 + }, + { + "KL/chosen_KL_mean": -19.98564910888672, + "KL/mean": -37.01270294189453, + "KL/rejected_KL_mean": -54.039756774902344, + "KL/std": 35.79534912109375, + "epoch": 0.16152716593245228, + "fcm_dpo/beta": 0.018443116918206215, + "fcm_dpo/delta": -0.2388836145401001, + "fcm_dpo/margin": 34.054107666015625, + "fcm_dpo/q_t": 0.3669545650482178, + "grad_norm": 22.622568130493164, + "learning_rate": 4.947482930773511e-07, + "logits/chosen": -0.6140676736831665, + "logits/rejected": -0.5725805163383484, + "logps/chosen": -78.73933410644531, + "logps/ref_chosen": -58.753684997558594, + "logps/ref_rejected": -79.75001525878906, + "logps/rejected": -133.78976440429688, + "loss": 0.9958, + "margin_dpo/margin_mean": 34.054107666015625, + "margin_dpo/margin_std": 42.70184326171875, + "step": 110 + }, + { + "KL/chosen_KL_mean": -23.330896377563477, + "KL/mean": -42.295196533203125, + "KL/rejected_KL_mean": -61.259498596191406, + "KL/std": 40.184547424316406, + "epoch": 0.16299559471365638, + "fcm_dpo/beta": 0.01799844577908516, + "fcm_dpo/delta": -0.29396289587020874, + "fcm_dpo/margin": 37.92860412597656, + "fcm_dpo/q_t": 0.35755258798599243, + "grad_norm": 26.07427978515625, + "learning_rate": 4.944834074412042e-07, + "logits/chosen": -0.6798889636993408, + "logits/rejected": -0.6572076082229614, + "logps/chosen": -91.95500183105469, + "logps/ref_chosen": -68.62410736083984, + "logps/ref_rejected": -98.42886352539062, + "logps/rejected": -159.6883544921875, + "loss": 0.979, + "margin_dpo/margin_mean": 37.92860794067383, + "margin_dpo/margin_std": 48.289306640625, + "step": 111 + }, + { + "KL/chosen_KL_mean": -21.652713775634766, + "KL/mean": -34.00275421142578, + "KL/rejected_KL_mean": -46.35279083251953, + "KL/std": 28.837787628173828, + "epoch": 0.1644640234948605, + "fcm_dpo/beta": 0.017818715423345566, + "fcm_dpo/delta": -0.04140431433916092, + "fcm_dpo/margin": 24.700084686279297, + "fcm_dpo/q_t": 0.40135613083839417, + "grad_norm": 22.577350616455078, + "learning_rate": 4.942120794399002e-07, + "logits/chosen": -0.6432993412017822, + "logits/rejected": -0.5998743772506714, + "logps/chosen": -71.90235900878906, + "logps/ref_chosen": -50.24964141845703, + "logps/ref_rejected": -64.77442932128906, + "logps/rejected": -111.1272201538086, + "loss": 1.0813, + "margin_dpo/margin_mean": 24.700084686279297, + "margin_dpo/margin_std": 34.30575180053711, + "step": 112 + }, + { + "KL/chosen_KL_mean": -26.370677947998047, + "KL/mean": -39.9312744140625, + "KL/rejected_KL_mean": -53.49187469482422, + "KL/std": 28.95420265197754, + "epoch": 0.16593245227606462, + "fcm_dpo/beta": 0.017736122012138367, + "fcm_dpo/delta": -0.08302031457424164, + "fcm_dpo/margin": 27.121187210083008, + "fcm_dpo/q_t": 0.39081066846847534, + "grad_norm": 22.69746208190918, + "learning_rate": 4.939343162231841e-07, + "logits/chosen": -0.6275451183319092, + "logits/rejected": -0.5815028548240662, + "logps/chosen": -93.08363342285156, + "logps/ref_chosen": -66.71295166015625, + "logps/ref_rejected": -77.96870422363281, + "logps/rejected": -131.4605712890625, + "loss": 1.0368, + "margin_dpo/margin_mean": 27.121185302734375, + "margin_dpo/margin_std": 32.2321891784668, + "step": 113 + }, + { + "KL/chosen_KL_mean": -24.161130905151367, + "KL/mean": -45.25044250488281, + "KL/rejected_KL_mean": -66.33973693847656, + "KL/std": 44.154300689697266, + "epoch": 0.16740088105726872, + "fcm_dpo/beta": 0.017194289714097977, + "fcm_dpo/delta": -0.3392394781112671, + "fcm_dpo/margin": 42.178619384765625, + "fcm_dpo/q_t": 0.3535785377025604, + "grad_norm": 23.69581413269043, + "learning_rate": 4.936501251103751e-07, + "logits/chosen": -0.6525863409042358, + "logits/rejected": -0.6185038089752197, + "logps/chosen": -81.94621276855469, + "logps/ref_chosen": -57.78507995605469, + "logps/ref_rejected": -87.10966491699219, + "logps/rejected": -153.44940185546875, + "loss": 0.9378, + "margin_dpo/margin_mean": 42.178619384765625, + "margin_dpo/margin_std": 53.02991485595703, + "step": 114 + }, + { + "KL/chosen_KL_mean": -32.144317626953125, + "KL/mean": -46.77862548828125, + "KL/rejected_KL_mean": -61.412940979003906, + "KL/std": 41.4689826965332, + "epoch": 0.16886930983847284, + "fcm_dpo/beta": 0.01700768992304802, + "fcm_dpo/delta": -0.1002306193113327, + "fcm_dpo/margin": 29.26862335205078, + "fcm_dpo/q_t": 0.3986138701438904, + "grad_norm": 32.29568862915039, + "learning_rate": 4.933595135901732e-07, + "logits/chosen": -0.6790868043899536, + "logits/rejected": -0.6547123193740845, + "logps/chosen": -97.72695922851562, + "logps/ref_chosen": -65.5826416015625, + "logps/ref_rejected": -98.56552124023438, + "logps/rejected": -159.97845458984375, + "loss": 1.1207, + "margin_dpo/margin_mean": 29.26862335205078, + "margin_dpo/margin_std": 53.58454895019531, + "step": 115 + }, + { + "KL/chosen_KL_mean": -24.143800735473633, + "KL/mean": -39.930686950683594, + "KL/rejected_KL_mean": -55.71758270263672, + "KL/std": 35.130592346191406, + "epoch": 0.17033773861967694, + "fcm_dpo/beta": 0.016804486513137817, + "fcm_dpo/delta": -0.13403692841529846, + "fcm_dpo/margin": 31.573772430419922, + "fcm_dpo/q_t": 0.38285309076309204, + "grad_norm": 24.514631271362305, + "learning_rate": 4.930624893204624e-07, + "logits/chosen": -0.64164799451828, + "logits/rejected": -0.6261519193649292, + "logps/chosen": -75.54411315917969, + "logps/ref_chosen": -51.40031433105469, + "logps/ref_rejected": -80.5218505859375, + "logps/rejected": -136.23944091796875, + "loss": 1.0119, + "margin_dpo/margin_mean": 31.573772430419922, + "margin_dpo/margin_std": 37.68627166748047, + "step": 116 + }, + { + "KL/chosen_KL_mean": -30.229568481445312, + "KL/mean": -43.88397216796875, + "KL/rejected_KL_mean": -57.53838348388672, + "KL/std": 37.56050491333008, + "epoch": 0.17180616740088106, + "fcm_dpo/beta": 0.01664135418832302, + "fcm_dpo/delta": -0.055726565420627594, + "fcm_dpo/margin": 27.30881118774414, + "fcm_dpo/q_t": 0.39999547600746155, + "grad_norm": 29.87828254699707, + "learning_rate": 4.927590601281083e-07, + "logits/chosen": -0.6171888113021851, + "logits/rejected": -0.5780812501907349, + "logps/chosen": -99.5279769897461, + "logps/ref_chosen": -69.29840850830078, + "logps/ref_rejected": -66.583984375, + "logps/rejected": -124.12237548828125, + "loss": 1.0906, + "margin_dpo/margin_mean": 27.30881118774414, + "margin_dpo/margin_std": 42.070045471191406, + "step": 117 + }, + { + "KL/chosen_KL_mean": -24.80445098876953, + "KL/mean": -40.591796875, + "KL/rejected_KL_mean": -56.37914276123047, + "KL/std": 33.962181091308594, + "epoch": 0.17327459618208516, + "fcm_dpo/beta": 0.016470149159431458, + "fcm_dpo/delta": -0.12322327494621277, + "fcm_dpo/margin": 31.574687957763672, + "fcm_dpo/q_t": 0.38508450984954834, + "grad_norm": 23.370370864868164, + "learning_rate": 4.924492340087524e-07, + "logits/chosen": -0.6490514278411865, + "logits/rejected": -0.6261621117591858, + "logps/chosen": -80.4454345703125, + "logps/ref_chosen": -55.6409797668457, + "logps/ref_rejected": -75.66905975341797, + "logps/rejected": -132.04820251464844, + "loss": 1.0199, + "margin_dpo/margin_mean": 31.574687957763672, + "margin_dpo/margin_std": 38.370567321777344, + "step": 118 + }, + { + "KL/chosen_KL_mean": -33.29602813720703, + "KL/mean": -48.99560546875, + "KL/rejected_KL_mean": -64.69517517089844, + "KL/std": 37.841270446777344, + "epoch": 0.17474302496328928, + "fcm_dpo/beta": 0.016222678124904633, + "fcm_dpo/delta": -0.11308414489030838, + "fcm_dpo/margin": 31.399154663085938, + "fcm_dpo/q_t": 0.39160895347595215, + "grad_norm": 25.578641891479492, + "learning_rate": 4.92133019126601e-07, + "logits/chosen": -0.6672037839889526, + "logits/rejected": -0.6525957584381104, + "logps/chosen": -106.80622863769531, + "logps/ref_chosen": -73.51019287109375, + "logps/ref_rejected": -102.977294921875, + "logps/rejected": -167.67247009277344, + "loss": 1.0591, + "margin_dpo/margin_mean": 31.399154663085938, + "margin_dpo/margin_std": 45.50007247924805, + "step": 119 + }, + { + "KL/chosen_KL_mean": -34.85655975341797, + "KL/mean": -56.261322021484375, + "KL/rejected_KL_mean": -77.66609191894531, + "KL/std": 47.26749801635742, + "epoch": 0.1762114537444934, + "fcm_dpo/beta": 0.015881016850471497, + "fcm_dpo/delta": -0.2897739112377167, + "fcm_dpo/margin": 42.80952453613281, + "fcm_dpo/q_t": 0.3552402853965759, + "grad_norm": 24.126750946044922, + "learning_rate": 4.918104238142103e-07, + "logits/chosen": -0.6502152681350708, + "logits/rejected": -0.6147547960281372, + "logps/chosen": -111.63739013671875, + "logps/ref_chosen": -76.78083801269531, + "logps/ref_rejected": -108.02374267578125, + "logps/rejected": -185.6898193359375, + "loss": 0.9457, + "margin_dpo/margin_mean": 42.80952453613281, + "margin_dpo/margin_std": 49.247589111328125, + "step": 120 + }, + { + "KL/chosen_KL_mean": -33.858795166015625, + "KL/mean": -57.48423767089844, + "KL/rejected_KL_mean": -81.10968017578125, + "KL/std": 46.71514892578125, + "epoch": 0.1776798825256975, + "fcm_dpo/beta": 0.015335135161876678, + "fcm_dpo/delta": -0.33797329664230347, + "fcm_dpo/margin": 47.250885009765625, + "fcm_dpo/q_t": 0.35153231024742126, + "grad_norm": 25.32863998413086, + "learning_rate": 4.91481456572267e-07, + "logits/chosen": -0.612008810043335, + "logits/rejected": -0.6054136753082275, + "logps/chosen": -95.64868927001953, + "logps/ref_chosen": -61.789894104003906, + "logps/ref_rejected": -109.99456787109375, + "logps/rejected": -191.104248046875, + "loss": 0.9492, + "margin_dpo/margin_mean": 47.250885009765625, + "margin_dpo/margin_std": 58.144596099853516, + "step": 121 + }, + { + "KL/chosen_KL_mean": -31.168033599853516, + "KL/mean": -60.511810302734375, + "KL/rejected_KL_mean": -89.85560607910156, + "KL/std": 53.64841842651367, + "epoch": 0.17914831130690162, + "fcm_dpo/beta": 0.014709394425153732, + "fcm_dpo/delta": -0.4840208888053894, + "fcm_dpo/margin": 58.68756103515625, + "fcm_dpo/q_t": 0.3186884820461273, + "grad_norm": 25.857219696044922, + "learning_rate": 4.911461260693638e-07, + "logits/chosen": -0.5904278755187988, + "logits/rejected": -0.6040619015693665, + "logps/chosen": -78.07024383544922, + "logps/ref_chosen": -46.9022102355957, + "logps/ref_rejected": -106.71418762207031, + "logps/rejected": -196.56979370117188, + "loss": 0.8285, + "margin_dpo/margin_mean": 58.68756103515625, + "margin_dpo/margin_std": 54.87202072143555, + "step": 122 + }, + { + "KL/chosen_KL_mean": -33.60801696777344, + "KL/mean": -53.090492248535156, + "KL/rejected_KL_mean": -72.57296752929688, + "KL/std": 46.223365783691406, + "epoch": 0.18061674008810572, + "fcm_dpo/beta": 0.014308318495750427, + "fcm_dpo/delta": -0.16279909014701843, + "fcm_dpo/margin": 38.964935302734375, + "fcm_dpo/q_t": 0.38503628969192505, + "grad_norm": 23.776098251342773, + "learning_rate": 4.908044411417711e-07, + "logits/chosen": -0.5932154655456543, + "logits/rejected": -0.5714948773384094, + "logps/chosen": -94.9466552734375, + "logps/ref_chosen": -61.33863830566406, + "logps/ref_rejected": -87.775390625, + "logps/rejected": -160.34835815429688, + "loss": 1.0682, + "margin_dpo/margin_mean": 38.964935302734375, + "margin_dpo/margin_std": 61.33662414550781, + "step": 123 + }, + { + "KL/chosen_KL_mean": -37.506778717041016, + "KL/mean": -65.6749267578125, + "KL/rejected_KL_mean": -93.84307861328125, + "KL/std": 61.69428253173828, + "epoch": 0.18208516886930984, + "fcm_dpo/beta": 0.013852743431925774, + "fcm_dpo/delta": -0.396537721157074, + "fcm_dpo/margin": 56.33629608154297, + "fcm_dpo/q_t": 0.35203444957733154, + "grad_norm": 27.827972412109375, + "learning_rate": 4.904564107932048e-07, + "logits/chosen": -0.5845953226089478, + "logits/rejected": -0.5840995907783508, + "logps/chosen": -108.95510864257812, + "logps/ref_chosen": -71.44833374023438, + "logps/ref_rejected": -117.58056640625, + "logps/rejected": -211.42364501953125, + "loss": 0.9658, + "margin_dpo/margin_mean": 56.33629608154297, + "margin_dpo/margin_std": 77.30990600585938, + "step": 124 + }, + { + "KL/chosen_KL_mean": -31.203296661376953, + "KL/mean": -56.6873779296875, + "KL/rejected_KL_mean": -82.17146301269531, + "KL/std": 51.79102325439453, + "epoch": 0.18355359765051396, + "fcm_dpo/beta": 0.013450113125145435, + "fcm_dpo/delta": -0.29537516832351685, + "fcm_dpo/margin": 50.968162536621094, + "fcm_dpo/q_t": 0.35877731442451477, + "grad_norm": 22.051082611083984, + "learning_rate": 4.90102044194588e-07, + "logits/chosen": -0.5767650604248047, + "logits/rejected": -0.5755800008773804, + "logps/chosen": -81.34024047851562, + "logps/ref_chosen": -50.136940002441406, + "logps/ref_rejected": -83.98861694335938, + "logps/rejected": -166.1600799560547, + "loss": 0.9621, + "margin_dpo/margin_mean": 50.968162536621094, + "margin_dpo/margin_std": 62.841583251953125, + "step": 125 + }, + { + "KL/chosen_KL_mean": -34.27449035644531, + "KL/mean": -57.64450454711914, + "KL/rejected_KL_mean": -81.01451110839844, + "KL/std": 46.706634521484375, + "epoch": 0.18502202643171806, + "fcm_dpo/beta": 0.013104382902383804, + "fcm_dpo/delta": -0.21946457028388977, + "fcm_dpo/margin": 46.74002456665039, + "fcm_dpo/q_t": 0.369617223739624, + "grad_norm": 23.553150177001953, + "learning_rate": 4.897413506838102e-07, + "logits/chosen": -0.5703192949295044, + "logits/rejected": -0.5591797232627869, + "logps/chosen": -89.94155883789062, + "logps/ref_chosen": -55.66706848144531, + "logps/ref_rejected": -98.1297607421875, + "logps/rejected": -179.14427185058594, + "loss": 0.9888, + "margin_dpo/margin_mean": 46.74002456665039, + "margin_dpo/margin_std": 58.35154724121094, + "step": 126 + }, + { + "KL/chosen_KL_mean": -32.794464111328125, + "KL/mean": -50.73387145996094, + "KL/rejected_KL_mean": -68.67326354980469, + "KL/std": 40.8072509765625, + "epoch": 0.18649045521292218, + "fcm_dpo/beta": 0.012982767075300217, + "fcm_dpo/delta": -0.0673581212759018, + "fcm_dpo/margin": 35.87880325317383, + "fcm_dpo/q_t": 0.3947966694831848, + "grad_norm": 23.849796295166016, + "learning_rate": 4.89374339765481e-07, + "logits/chosen": -0.579119086265564, + "logits/rejected": -0.5568236112594604, + "logps/chosen": -89.3491439819336, + "logps/ref_chosen": -56.55467987060547, + "logps/ref_rejected": -76.7957763671875, + "logps/rejected": -145.46905517578125, + "loss": 1.073, + "margin_dpo/margin_mean": 35.87880325317383, + "margin_dpo/margin_std": 50.529823303222656, + "step": 127 + }, + { + "KL/chosen_KL_mean": -35.758731842041016, + "KL/mean": -54.428401947021484, + "KL/rejected_KL_mean": -73.09808349609375, + "KL/std": 48.978797912597656, + "epoch": 0.18795888399412627, + "fcm_dpo/beta": 0.012904481962323189, + "fcm_dpo/delta": -0.08387961983680725, + "fcm_dpo/margin": 37.3393440246582, + "fcm_dpo/q_t": 0.3962070941925049, + "grad_norm": 35.115657806396484, + "learning_rate": 4.890010211106795e-07, + "logits/chosen": -0.5600550174713135, + "logits/rejected": -0.5325770378112793, + "logps/chosen": -93.87968444824219, + "logps/ref_chosen": -58.12095642089844, + "logps/ref_rejected": -76.43896484375, + "logps/rejected": -149.53704833984375, + "loss": 1.0968, + "margin_dpo/margin_mean": 37.3393440246582, + "margin_dpo/margin_std": 60.14480972290039, + "step": 128 + }, + { + "KL/chosen_KL_mean": -44.086055755615234, + "KL/mean": -63.96501922607422, + "KL/rejected_KL_mean": -83.84398651123047, + "KL/std": 53.29756164550781, + "epoch": 0.1894273127753304, + "fcm_dpo/beta": 0.012750450521707535, + "fcm_dpo/delta": -0.10966508090496063, + "fcm_dpo/margin": 39.75792694091797, + "fcm_dpo/q_t": 0.396596223115921, + "grad_norm": 24.342992782592773, + "learning_rate": 4.88621404556699e-07, + "logits/chosen": -0.5686518549919128, + "logits/rejected": -0.5536836385726929, + "logps/chosen": -111.00242614746094, + "logps/ref_chosen": -66.91637420654297, + "logps/ref_rejected": -96.6422119140625, + "logps/rejected": -180.4862060546875, + "loss": 1.1002, + "margin_dpo/margin_mean": 39.75792694091797, + "margin_dpo/margin_std": 68.46754455566406, + "step": 129 + }, + { + "KL/chosen_KL_mean": -33.01520919799805, + "KL/mean": -62.57740020751953, + "KL/rejected_KL_mean": -92.13957977294922, + "KL/std": 57.29778289794922, + "epoch": 0.19089574155653452, + "fcm_dpo/beta": 0.012459097430109978, + "fcm_dpo/delta": -0.34889647364616394, + "fcm_dpo/margin": 59.1243782043457, + "fcm_dpo/q_t": 0.34880900382995605, + "grad_norm": 21.42711639404297, + "learning_rate": 4.882355001067891e-07, + "logits/chosen": -0.5372148752212524, + "logits/rejected": -0.5315117835998535, + "logps/chosen": -77.68206024169922, + "logps/ref_chosen": -44.66685104370117, + "logps/ref_rejected": -82.78165435791016, + "logps/rejected": -174.92123413085938, + "loss": 0.9393, + "margin_dpo/margin_mean": 59.1243782043457, + "margin_dpo/margin_std": 68.27285766601562, + "step": 130 + }, + { + "KL/chosen_KL_mean": -28.63924217224121, + "KL/mean": -57.72308349609375, + "KL/rejected_KL_mean": -86.80691528320312, + "KL/std": 54.85826110839844, + "epoch": 0.19236417033773862, + "fcm_dpo/beta": 0.011984329670667648, + "fcm_dpo/delta": -0.30810654163360596, + "fcm_dpo/margin": 58.167686462402344, + "fcm_dpo/q_t": 0.3473639190196991, + "grad_norm": 27.132959365844727, + "learning_rate": 4.878433179298909e-07, + "logits/chosen": -0.5333915948867798, + "logits/rejected": -0.5392862558364868, + "logps/chosen": -73.56382751464844, + "logps/ref_chosen": -44.924591064453125, + "logps/ref_rejected": -88.44401550292969, + "logps/rejected": -175.2509307861328, + "loss": 0.9189, + "margin_dpo/margin_mean": 58.167686462402344, + "margin_dpo/margin_std": 60.25544738769531, + "step": 131 + }, + { + "KL/chosen_KL_mean": -38.82572937011719, + "KL/mean": -63.184059143066406, + "KL/rejected_KL_mean": -87.54237365722656, + "KL/std": 55.58538818359375, + "epoch": 0.19383259911894274, + "fcm_dpo/beta": 0.011730792000889778, + "fcm_dpo/delta": -0.1767834722995758, + "fcm_dpo/margin": 48.716651916503906, + "fcm_dpo/q_t": 0.3818228244781494, + "grad_norm": 23.049833297729492, + "learning_rate": 4.874448683603694e-07, + "logits/chosen": -0.5593503713607788, + "logits/rejected": -0.5511766672134399, + "logps/chosen": -97.82681274414062, + "logps/ref_chosen": -59.00108337402344, + "logps/ref_rejected": -87.89215087890625, + "logps/rejected": -175.43453979492188, + "loss": 1.034, + "margin_dpo/margin_mean": 48.716651916503906, + "margin_dpo/margin_std": 71.09864807128906, + "step": 132 + }, + { + "KL/chosen_KL_mean": -45.97046661376953, + "KL/mean": -67.64411163330078, + "KL/rejected_KL_mean": -89.3177490234375, + "KL/std": 51.025794982910156, + "epoch": 0.19530102790014683, + "fcm_dpo/beta": 0.011588573455810547, + "fcm_dpo/delta": -0.1051153689622879, + "fcm_dpo/margin": 43.3472900390625, + "fcm_dpo/q_t": 0.3930418789386749, + "grad_norm": 26.683420181274414, + "learning_rate": 4.870401618977415e-07, + "logits/chosen": -0.5510296821594238, + "logits/rejected": -0.5348105430603027, + "logps/chosen": -112.57496643066406, + "logps/ref_chosen": -66.60449981689453, + "logps/ref_rejected": -96.33355712890625, + "logps/rejected": -185.65130615234375, + "loss": 1.0613, + "margin_dpo/margin_mean": 43.3472900390625, + "margin_dpo/margin_std": 63.578800201416016, + "step": 133 + }, + { + "KL/chosen_KL_mean": -36.70982360839844, + "KL/mean": -60.16697311401367, + "KL/rejected_KL_mean": -83.62411499023438, + "KL/std": 50.47258758544922, + "epoch": 0.19676945668135096, + "fcm_dpo/beta": 0.011467832140624523, + "fcm_dpo/delta": -0.14169654250144958, + "fcm_dpo/margin": 46.91429901123047, + "fcm_dpo/q_t": 0.38246801495552063, + "grad_norm": 22.107864379882812, + "learning_rate": 4.866292092063986e-07, + "logits/chosen": -0.5395177006721497, + "logits/rejected": -0.5207037329673767, + "logps/chosen": -88.77908325195312, + "logps/ref_chosen": -52.06925582885742, + "logps/ref_rejected": -87.6545181274414, + "logps/rejected": -171.27862548828125, + "loss": 1.0101, + "margin_dpo/margin_mean": 46.91429901123047, + "margin_dpo/margin_std": 57.2902946472168, + "step": 134 + }, + { + "KL/chosen_KL_mean": -41.4932861328125, + "KL/mean": -73.72633361816406, + "KL/rejected_KL_mean": -105.95936584472656, + "KL/std": 65.24856567382812, + "epoch": 0.19823788546255505, + "fcm_dpo/beta": 0.011130950413644314, + "fcm_dpo/delta": -0.3293672502040863, + "fcm_dpo/margin": 64.46609497070312, + "fcm_dpo/q_t": 0.35333138704299927, + "grad_norm": 23.30121612548828, + "learning_rate": 4.862120211153265e-07, + "logits/chosen": -0.5058168768882751, + "logits/rejected": -0.5387102961540222, + "logps/chosen": -91.84715270996094, + "logps/ref_chosen": -50.353858947753906, + "logps/ref_rejected": -115.97975158691406, + "logps/rejected": -221.93911743164062, + "loss": 0.9469, + "margin_dpo/margin_mean": 64.46609497070312, + "margin_dpo/margin_std": 78.12184143066406, + "step": 135 + }, + { + "KL/chosen_KL_mean": -48.40357208251953, + "KL/mean": -70.24093627929688, + "KL/rejected_KL_mean": -92.07830810546875, + "KL/std": 58.511390686035156, + "epoch": 0.19970631424375918, + "fcm_dpo/beta": 0.010931683704257011, + "fcm_dpo/delta": -0.08014538139104843, + "fcm_dpo/margin": 43.674739837646484, + "fcm_dpo/q_t": 0.4048768877983093, + "grad_norm": 23.915916442871094, + "learning_rate": 4.857886086178193e-07, + "logits/chosen": -0.5324341654777527, + "logits/rejected": -0.5198547840118408, + "logps/chosen": -113.47608184814453, + "logps/ref_chosen": -65.072509765625, + "logps/ref_rejected": -96.32122802734375, + "logps/rejected": -188.3995361328125, + "loss": 1.1016, + "margin_dpo/margin_mean": 43.67473602294922, + "margin_dpo/margin_std": 73.63079833984375, + "step": 136 + }, + { + "KL/chosen_KL_mean": -48.21935272216797, + "KL/mean": -82.04722595214844, + "KL/rejected_KL_mean": -115.87509155273438, + "KL/std": 78.75631713867188, + "epoch": 0.2011747430249633, + "fcm_dpo/beta": 0.010671773925423622, + "fcm_dpo/delta": -0.3343368172645569, + "fcm_dpo/margin": 67.65573120117188, + "fcm_dpo/q_t": 0.35995662212371826, + "grad_norm": 21.44792938232422, + "learning_rate": 4.853589828711902e-07, + "logits/chosen": -0.49152958393096924, + "logits/rejected": -0.5171319246292114, + "logps/chosen": -96.97846984863281, + "logps/ref_chosen": -48.759117126464844, + "logps/ref_rejected": -113.86376953125, + "logps/rejected": -229.73886108398438, + "loss": 0.9823, + "margin_dpo/margin_mean": 67.65573120117188, + "margin_dpo/margin_std": 93.46538543701172, + "step": 137 + }, + { + "KL/chosen_KL_mean": -49.967342376708984, + "KL/mean": -75.7919692993164, + "KL/rejected_KL_mean": -101.61659240722656, + "KL/std": 60.058433532714844, + "epoch": 0.2026431718061674, + "fcm_dpo/beta": 0.010484375059604645, + "fcm_dpo/delta": -0.14534850418567657, + "fcm_dpo/margin": 51.649253845214844, + "fcm_dpo/q_t": 0.37925344705581665, + "grad_norm": 23.9995174407959, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": -0.4553467631340027, + "logits/rejected": -0.4406394362449646, + "logps/chosen": -110.48699188232422, + "logps/ref_chosen": -60.519649505615234, + "logps/ref_rejected": -93.19694519042969, + "logps/rejected": -194.81353759765625, + "loss": 1.0095, + "margin_dpo/margin_mean": 51.649253845214844, + "margin_dpo/margin_std": 61.569252014160156, + "step": 138 + }, + { + "KL/chosen_KL_mean": -40.474876403808594, + "KL/mean": -71.97026062011719, + "KL/rejected_KL_mean": -103.46565246582031, + "KL/std": 57.427764892578125, + "epoch": 0.20411160058737152, + "fcm_dpo/beta": 0.010253118351101875, + "fcm_dpo/delta": -0.25380727648735046, + "fcm_dpo/margin": 62.99077224731445, + "fcm_dpo/q_t": 0.3614668846130371, + "grad_norm": 20.68395233154297, + "learning_rate": 4.844811370781446e-07, + "logits/chosen": -0.5000328421592712, + "logits/rejected": -0.48780351877212524, + "logps/chosen": -87.36625671386719, + "logps/ref_chosen": -46.89138412475586, + "logps/ref_rejected": -79.72798156738281, + "logps/rejected": -183.19363403320312, + "loss": 0.9557, + "margin_dpo/margin_mean": 62.99076843261719, + "margin_dpo/margin_std": 72.46928405761719, + "step": 139 + }, + { + "KL/chosen_KL_mean": -49.33660125732422, + "KL/mean": -78.02191925048828, + "KL/rejected_KL_mean": -106.70724487304688, + "KL/std": 64.60366821289062, + "epoch": 0.2055800293685756, + "fcm_dpo/beta": 0.010033691301941872, + "fcm_dpo/delta": -0.1809324026107788, + "fcm_dpo/margin": 57.37063980102539, + "fcm_dpo/q_t": 0.37695828080177307, + "grad_norm": 23.712533950805664, + "learning_rate": 4.840329401637809e-07, + "logits/chosen": -0.4792005717754364, + "logits/rejected": -0.46222788095474243, + "logps/chosen": -108.31130981445312, + "logps/ref_chosen": -58.97471618652344, + "logps/ref_rejected": -83.28410339355469, + "logps/rejected": -189.99134826660156, + "loss": 1.0174, + "margin_dpo/margin_mean": 57.370643615722656, + "margin_dpo/margin_std": 76.40518188476562, + "step": 140 + }, + { + "KL/chosen_KL_mean": -53.7377815246582, + "KL/mean": -81.11244201660156, + "KL/rejected_KL_mean": -108.48710632324219, + "KL/std": 67.55751037597656, + "epoch": 0.20704845814977973, + "fcm_dpo/beta": 0.009888198226690292, + "fcm_dpo/delta": -0.14525547623634338, + "fcm_dpo/margin": 54.74932098388672, + "fcm_dpo/q_t": 0.3810844421386719, + "grad_norm": 26.525779724121094, + "learning_rate": 4.83578576263792e-07, + "logits/chosen": -0.4832468628883362, + "logits/rejected": -0.46669474244117737, + "logps/chosen": -128.81344604492188, + "logps/ref_chosen": -75.07566833496094, + "logps/ref_rejected": -98.1922607421875, + "logps/rejected": -206.6793670654297, + "loss": 1.0432, + "margin_dpo/margin_mean": 54.74932098388672, + "margin_dpo/margin_std": 76.88491821289062, + "step": 141 + }, + { + "KL/chosen_KL_mean": -56.92424774169922, + "KL/mean": -88.21382141113281, + "KL/rejected_KL_mean": -119.50340270996094, + "KL/std": 76.11981201171875, + "epoch": 0.20851688693098386, + "fcm_dpo/beta": 0.009715566411614418, + "fcm_dpo/delta": -0.2142324447631836, + "fcm_dpo/margin": 62.57915496826172, + "fcm_dpo/q_t": 0.3745476007461548, + "grad_norm": 27.3022403717041, + "learning_rate": 4.83118057351089e-07, + "logits/chosen": -0.4715278446674347, + "logits/rejected": -0.4681225121021271, + "logps/chosen": -114.95217895507812, + "logps/ref_chosen": -58.027931213378906, + "logps/ref_rejected": -94.58222961425781, + "logps/rejected": -214.08563232421875, + "loss": 1.0324, + "margin_dpo/margin_mean": 62.57914733886719, + "margin_dpo/margin_std": 89.07696533203125, + "step": 142 + }, + { + "KL/chosen_KL_mean": -61.330406188964844, + "KL/mean": -81.28429412841797, + "KL/rejected_KL_mean": -101.2381820678711, + "KL/std": 67.98274230957031, + "epoch": 0.20998531571218795, + "fcm_dpo/beta": 0.009628836065530777, + "fcm_dpo/delta": 0.015870332717895508, + "fcm_dpo/margin": 39.907772064208984, + "fcm_dpo/q_t": 0.4193767309188843, + "grad_norm": 26.948379516601562, + "learning_rate": 4.826513955607734e-07, + "logits/chosen": -0.42380428314208984, + "logits/rejected": -0.4101778268814087, + "logps/chosen": -118.9268569946289, + "logps/ref_chosen": -57.59645080566406, + "logps/ref_rejected": -78.99957275390625, + "logps/rejected": -180.23776245117188, + "loss": 1.1652, + "margin_dpo/margin_mean": 39.907772064208984, + "margin_dpo/margin_std": 78.14988708496094, + "step": 143 + }, + { + "KL/chosen_KL_mean": -52.056514739990234, + "KL/mean": -77.15028381347656, + "KL/rejected_KL_mean": -102.24404907226562, + "KL/std": 56.15443420410156, + "epoch": 0.21145374449339208, + "fcm_dpo/beta": 0.009588293731212616, + "fcm_dpo/delta": -0.08328879624605179, + "fcm_dpo/margin": 50.187538146972656, + "fcm_dpo/q_t": 0.39288330078125, + "grad_norm": 22.615461349487305, + "learning_rate": 4.821786031898176e-07, + "logits/chosen": -0.45128071308135986, + "logits/rejected": -0.42970043420791626, + "logps/chosen": -111.96287536621094, + "logps/ref_chosen": -59.90636444091797, + "logps/ref_rejected": -82.00025939941406, + "logps/rejected": -184.24432373046875, + "loss": 1.0523, + "margin_dpo/margin_mean": 50.187538146972656, + "margin_dpo/margin_std": 65.81391143798828, + "step": 144 + }, + { + "KL/chosen_KL_mean": -52.327789306640625, + "KL/mean": -79.01912689208984, + "KL/rejected_KL_mean": -105.71046447753906, + "KL/std": 58.76500701904297, + "epoch": 0.21292217327459617, + "fcm_dpo/beta": 0.009492561221122742, + "fcm_dpo/delta": -0.10956543684005737, + "fcm_dpo/margin": 53.38267517089844, + "fcm_dpo/q_t": 0.3878716230392456, + "grad_norm": 26.634552001953125, + "learning_rate": 4.816996926967401e-07, + "logits/chosen": -0.4835849404335022, + "logits/rejected": -0.46149128675460815, + "logps/chosen": -108.92845153808594, + "logps/ref_chosen": -56.60066604614258, + "logps/ref_rejected": -77.86631774902344, + "logps/rejected": -183.5767822265625, + "loss": 1.0402, + "margin_dpo/margin_mean": 53.38267517089844, + "margin_dpo/margin_std": 69.78041076660156, + "step": 145 + }, + { + "KL/chosen_KL_mean": -71.99324035644531, + "KL/mean": -92.98985290527344, + "KL/rejected_KL_mean": -113.98645782470703, + "KL/std": 62.72893524169922, + "epoch": 0.2143906020558003, + "fcm_dpo/beta": 0.009455716237425804, + "fcm_dpo/delta": 0.002865489572286606, + "fcm_dpo/margin": 41.99322509765625, + "fcm_dpo/q_t": 0.41135260462760925, + "grad_norm": 31.551198959350586, + "learning_rate": 4.812146767012779e-07, + "logits/chosen": -0.4793580174446106, + "logits/rejected": -0.44883793592453003, + "logps/chosen": -137.99368286132812, + "logps/ref_chosen": -66.00045013427734, + "logps/ref_rejected": -81.70278930664062, + "logps/rejected": -195.68923950195312, + "loss": 1.1451, + "margin_dpo/margin_mean": 41.99322509765625, + "margin_dpo/margin_std": 74.72547912597656, + "step": 146 + }, + { + "KL/chosen_KL_mean": -52.66679000854492, + "KL/mean": -81.16831970214844, + "KL/rejected_KL_mean": -109.66986083984375, + "KL/std": 64.48698425292969, + "epoch": 0.21585903083700442, + "fcm_dpo/beta": 0.009364601224660873, + "fcm_dpo/delta": -0.13770800828933716, + "fcm_dpo/margin": 57.00305938720703, + "fcm_dpo/q_t": 0.38518407940864563, + "grad_norm": 22.39113426208496, + "learning_rate": 4.807235679840536e-07, + "logits/chosen": -0.4767155647277832, + "logits/rejected": -0.4504894018173218, + "logps/chosen": -106.07228088378906, + "logps/ref_chosen": -53.405487060546875, + "logps/ref_rejected": -71.39060974121094, + "logps/rejected": -181.0604705810547, + "loss": 1.042, + "margin_dpo/margin_mean": 57.0030632019043, + "margin_dpo/margin_std": 79.139892578125, + "step": 147 + }, + { + "KL/chosen_KL_mean": -51.239707946777344, + "KL/mean": -74.8543701171875, + "KL/rejected_KL_mean": -98.46903991699219, + "KL/std": 62.515411376953125, + "epoch": 0.2173274596182085, + "fcm_dpo/beta": 0.009360980242490768, + "fcm_dpo/delta": -0.043911248445510864, + "fcm_dpo/margin": 47.22932434082031, + "fcm_dpo/q_t": 0.40432801842689514, + "grad_norm": 21.143596649169922, + "learning_rate": 4.802263794862384e-07, + "logits/chosen": -0.5241053104400635, + "logits/rejected": -0.5143436789512634, + "logps/chosen": -116.17678833007812, + "logps/ref_chosen": -64.93708038330078, + "logps/ref_rejected": -103.09384155273438, + "logps/rejected": -201.56288146972656, + "loss": 1.0885, + "margin_dpo/margin_mean": 47.22932434082031, + "margin_dpo/margin_std": 65.89462280273438, + "step": 148 + }, + { + "KL/chosen_KL_mean": -49.8416633605957, + "KL/mean": -79.41673278808594, + "KL/rejected_KL_mean": -108.9917984008789, + "KL/std": 57.800533294677734, + "epoch": 0.21879588839941264, + "fcm_dpo/beta": 0.009160520508885384, + "fcm_dpo/delta": -0.14721731841564178, + "fcm_dpo/margin": 59.15013122558594, + "fcm_dpo/q_t": 0.3789201080799103, + "grad_norm": 20.35516357421875, + "learning_rate": 4.797231243092118e-07, + "logits/chosen": -0.5416568517684937, + "logits/rejected": -0.5244969129562378, + "logps/chosen": -108.3154296875, + "logps/ref_chosen": -58.47376251220703, + "logps/ref_rejected": -99.31474304199219, + "logps/rejected": -208.30654907226562, + "loss": 1.0075, + "margin_dpo/margin_mean": 59.150123596191406, + "margin_dpo/margin_std": 67.78451538085938, + "step": 149 + }, + { + "KL/chosen_KL_mean": -44.17631912231445, + "KL/mean": -72.84591674804688, + "KL/rejected_KL_mean": -101.5155029296875, + "KL/std": 67.9078598022461, + "epoch": 0.22026431718061673, + "fcm_dpo/beta": 0.009045644663274288, + "fcm_dpo/delta": -0.12300632148981094, + "fcm_dpo/margin": 57.33917999267578, + "fcm_dpo/q_t": 0.3912937641143799, + "grad_norm": 20.950302124023438, + "learning_rate": 4.792138157142157e-07, + "logits/chosen": -0.472980797290802, + "logits/rejected": -0.4736313223838806, + "logps/chosen": -89.88212585449219, + "logps/ref_chosen": -45.705810546875, + "logps/ref_rejected": -83.34759521484375, + "logps/rejected": -184.86309814453125, + "loss": 1.0437, + "margin_dpo/margin_mean": 57.33917999267578, + "margin_dpo/margin_std": 80.5770034790039, + "step": 150 + }, + { + "KL/chosen_KL_mean": -54.050907135009766, + "KL/mean": -83.80128479003906, + "KL/rejected_KL_mean": -113.55165100097656, + "KL/std": 63.99048614501953, + "epoch": 0.22173274596182085, + "fcm_dpo/beta": 0.008962772786617279, + "fcm_dpo/delta": -0.13693660497665405, + "fcm_dpo/margin": 59.50074768066406, + "fcm_dpo/q_t": 0.38013043999671936, + "grad_norm": 24.177379608154297, + "learning_rate": 4.786984671220053e-07, + "logits/chosen": -0.5812417268753052, + "logits/rejected": -0.5512596368789673, + "logps/chosen": -124.62174224853516, + "logps/ref_chosen": -70.57083129882812, + "logps/ref_rejected": -100.46382141113281, + "logps/rejected": -214.01547241210938, + "loss": 1.0088, + "margin_dpo/margin_mean": 59.50074768066406, + "margin_dpo/margin_std": 69.27177429199219, + "step": 151 + }, + { + "KL/chosen_KL_mean": -47.178958892822266, + "KL/mean": -82.26052856445312, + "KL/rejected_KL_mean": -117.34210968017578, + "KL/std": 66.30995178222656, + "epoch": 0.22320117474302498, + "fcm_dpo/beta": 0.008797157555818558, + "fcm_dpo/delta": -0.22390428185462952, + "fcm_dpo/margin": 70.16314697265625, + "fcm_dpo/q_t": 0.3667450547218323, + "grad_norm": 22.298110961914062, + "learning_rate": 4.78177092112495e-07, + "logits/chosen": -0.5535380840301514, + "logits/rejected": -0.5499355792999268, + "logps/chosen": -107.34334564208984, + "logps/ref_chosen": -60.16438674926758, + "logps/ref_rejected": -106.14045715332031, + "logps/rejected": -223.48257446289062, + "loss": 0.9731, + "margin_dpo/margin_mean": 70.16314697265625, + "margin_dpo/margin_std": 80.07929992675781, + "step": 152 + }, + { + "KL/chosen_KL_mean": -48.634864807128906, + "KL/mean": -78.62088012695312, + "KL/rejected_KL_mean": -108.60690307617188, + "KL/std": 71.24372863769531, + "epoch": 0.22466960352422907, + "fcm_dpo/beta": 0.008663169108331203, + "fcm_dpo/delta": -0.12265150249004364, + "fcm_dpo/margin": 59.97203063964844, + "fcm_dpo/q_t": 0.38973453640937805, + "grad_norm": 17.89531898498535, + "learning_rate": 4.776497044244016e-07, + "logits/chosen": -0.5327095985412598, + "logits/rejected": -0.5256816744804382, + "logps/chosen": -104.95014190673828, + "logps/ref_chosen": -56.315277099609375, + "logps/ref_rejected": -85.65583801269531, + "logps/rejected": -194.26272583007812, + "loss": 1.0512, + "margin_dpo/margin_mean": 59.97203063964844, + "margin_dpo/margin_std": 86.57862854003906, + "step": 153 + }, + { + "KL/chosen_KL_mean": -57.993751525878906, + "KL/mean": -87.43338775634766, + "KL/rejected_KL_mean": -116.87301635742188, + "KL/std": 72.61052703857422, + "epoch": 0.2261380323054332, + "fcm_dpo/beta": 0.008578930050134659, + "fcm_dpo/delta": -0.10782384127378464, + "fcm_dpo/margin": 58.87928009033203, + "fcm_dpo/q_t": 0.3921113610267639, + "grad_norm": 21.391340255737305, + "learning_rate": 4.771163179548808e-07, + "logits/chosen": -0.4942118525505066, + "logits/rejected": -0.492919921875, + "logps/chosen": -120.73632049560547, + "logps/ref_chosen": -62.74256896972656, + "logps/ref_rejected": -104.24420166015625, + "logps/rejected": -221.11721801757812, + "loss": 1.0787, + "margin_dpo/margin_mean": 58.87928009033203, + "margin_dpo/margin_std": 90.07198333740234, + "step": 154 + }, + { + "KL/chosen_KL_mean": -52.702293395996094, + "KL/mean": -82.81259155273438, + "KL/rejected_KL_mean": -112.92288208007812, + "KL/std": 68.29090118408203, + "epoch": 0.2276064610866373, + "fcm_dpo/beta": 0.0084763802587986, + "fcm_dpo/delta": -0.11326827853918076, + "fcm_dpo/margin": 60.2205924987793, + "fcm_dpo/q_t": 0.38809964060783386, + "grad_norm": 21.07434844970703, + "learning_rate": 4.7657694675916247e-07, + "logits/chosen": -0.5273807048797607, + "logits/rejected": -0.5057998895645142, + "logps/chosen": -113.35547637939453, + "logps/ref_chosen": -60.65318298339844, + "logps/ref_rejected": -77.49220275878906, + "logps/rejected": -190.4150848388672, + "loss": 1.0476, + "margin_dpo/margin_mean": 60.22058868408203, + "margin_dpo/margin_std": 82.90054321289062, + "step": 155 + }, + { + "KL/chosen_KL_mean": -74.92457580566406, + "KL/mean": -92.70379638671875, + "KL/rejected_KL_mean": -110.48301696777344, + "KL/std": 70.93622589111328, + "epoch": 0.2290748898678414, + "fcm_dpo/beta": 0.008492467924952507, + "fcm_dpo/delta": 0.09958405047655106, + "fcm_dpo/margin": 35.558448791503906, + "fcm_dpo/q_t": 0.43309691548347473, + "grad_norm": 32.95204162597656, + "learning_rate": 4.7603160505017893e-07, + "logits/chosen": -0.4810073971748352, + "logits/rejected": -0.468288779258728, + "logps/chosen": -144.41644287109375, + "logps/ref_chosen": -69.49188232421875, + "logps/ref_rejected": -77.16929626464844, + "logps/rejected": -187.65231323242188, + "loss": 1.252, + "margin_dpo/margin_mean": 35.55845260620117, + "margin_dpo/margin_std": 93.71145629882812, + "step": 156 + }, + { + "KL/chosen_KL_mean": -68.62439727783203, + "KL/mean": -105.9276123046875, + "KL/rejected_KL_mean": -143.2308349609375, + "KL/std": 78.11910247802734, + "epoch": 0.2305433186490455, + "fcm_dpo/beta": 0.008352255448698997, + "fcm_dpo/delta": -0.23134897649288177, + "fcm_dpo/margin": 74.60643768310547, + "fcm_dpo/q_t": 0.3620302677154541, + "grad_norm": 25.482498168945312, + "learning_rate": 4.7548030719819154e-07, + "logits/chosen": -0.43237775564193726, + "logits/rejected": -0.4356487989425659, + "logps/chosen": -129.99282836914062, + "logps/ref_chosen": -61.368438720703125, + "logps/ref_rejected": -107.64636993408203, + "logps/rejected": -250.877197265625, + "loss": 0.9816, + "margin_dpo/margin_mean": 74.6064453125, + "margin_dpo/margin_std": 87.3945541381836, + "step": 157 + }, + { + "KL/chosen_KL_mean": -68.26705932617188, + "KL/mean": -107.6445083618164, + "KL/rejected_KL_mean": -147.02197265625, + "KL/std": 97.27963256835938, + "epoch": 0.23201174743024963, + "fcm_dpo/beta": 0.008176269009709358, + "fcm_dpo/delta": -0.2519432008266449, + "fcm_dpo/margin": 78.75489807128906, + "fcm_dpo/q_t": 0.37355363368988037, + "grad_norm": 22.437089920043945, + "learning_rate": 4.7492306773041136e-07, + "logits/chosen": -0.41110551357269287, + "logits/rejected": -0.4253670275211334, + "logps/chosen": -125.87997436523438, + "logps/ref_chosen": -57.612918853759766, + "logps/ref_rejected": -113.6946792602539, + "logps/rejected": -260.7166442871094, + "loss": 1.0189, + "margin_dpo/margin_mean": 78.75489807128906, + "margin_dpo/margin_std": 116.05311584472656, + "step": 158 + }, + { + "KL/chosen_KL_mean": -77.33633422851562, + "KL/mean": -105.33822631835938, + "KL/rejected_KL_mean": -133.34011840820312, + "KL/std": 85.79032897949219, + "epoch": 0.23348017621145375, + "fcm_dpo/beta": 0.008105665445327759, + "fcm_dpo/delta": -0.05528353154659271, + "fcm_dpo/margin": 56.00377655029297, + "fcm_dpo/q_t": 0.40175575017929077, + "grad_norm": 24.443878173828125, + "learning_rate": 4.743599013306165e-07, + "logits/chosen": -0.46176522970199585, + "logits/rejected": -0.42901933193206787, + "logps/chosen": -158.89666748046875, + "logps/ref_chosen": -81.56034851074219, + "logps/ref_rejected": -88.89871215820312, + "logps/rejected": -222.23883056640625, + "loss": 1.1099, + "margin_dpo/margin_mean": 56.00377655029297, + "margin_dpo/margin_std": 92.10934448242188, + "step": 159 + }, + { + "KL/chosen_KL_mean": -79.1042251586914, + "KL/mean": -115.31195068359375, + "KL/rejected_KL_mean": -151.51966857910156, + "KL/std": 91.41381072998047, + "epoch": 0.23494860499265785, + "fcm_dpo/beta": 0.007949615828692913, + "fcm_dpo/delta": -0.18148598074913025, + "fcm_dpo/margin": 72.41543579101562, + "fcm_dpo/q_t": 0.38441920280456543, + "grad_norm": 28.788623809814453, + "learning_rate": 4.737908228387656e-07, + "logits/chosen": -0.4013976454734802, + "logits/rejected": -0.38853079080581665, + "logps/chosen": -144.83511352539062, + "logps/ref_chosen": -65.73088073730469, + "logps/ref_rejected": -97.21781921386719, + "logps/rejected": -248.73748779296875, + "loss": 1.0617, + "margin_dpo/margin_mean": 72.41543579101562, + "margin_dpo/margin_std": 113.82682037353516, + "step": 160 + }, + { + "KL/chosen_KL_mean": -68.4842529296875, + "KL/mean": -100.51941680908203, + "KL/rejected_KL_mean": -132.55458068847656, + "KL/std": 72.564453125, + "epoch": 0.23641703377386197, + "fcm_dpo/beta": 0.007867861539125443, + "fcm_dpo/delta": -0.10678651928901672, + "fcm_dpo/margin": 64.07034301757812, + "fcm_dpo/q_t": 0.3917103111743927, + "grad_norm": 26.057464599609375, + "learning_rate": 4.7321584725060594e-07, + "logits/chosen": -0.41672807931900024, + "logits/rejected": -0.41233956813812256, + "logps/chosen": -120.92071533203125, + "logps/ref_chosen": -52.43647003173828, + "logps/ref_rejected": -83.43095397949219, + "logps/rejected": -215.98553466796875, + "loss": 1.0604, + "margin_dpo/margin_mean": 64.07034301757812, + "margin_dpo/margin_std": 92.08967590332031, + "step": 161 + }, + { + "KL/chosen_KL_mean": -64.87035369873047, + "KL/mean": -97.34904479980469, + "KL/rejected_KL_mean": -129.82774353027344, + "KL/std": 81.12202453613281, + "epoch": 0.23788546255506607, + "fcm_dpo/beta": 0.0077486904338002205, + "fcm_dpo/delta": -0.1073184385895729, + "fcm_dpo/margin": 64.9573974609375, + "fcm_dpo/q_t": 0.39209944009780884, + "grad_norm": 23.717235565185547, + "learning_rate": 4.7263498971727905e-07, + "logits/chosen": -0.46293187141418457, + "logits/rejected": -0.44232678413391113, + "logps/chosen": -127.48094177246094, + "logps/ref_chosen": -62.6105842590332, + "logps/ref_rejected": -89.39057922363281, + "logps/rejected": -219.21832275390625, + "loss": 1.0674, + "margin_dpo/margin_mean": 64.9573974609375, + "margin_dpo/margin_std": 94.93345642089844, + "step": 162 + }, + { + "KL/chosen_KL_mean": -73.33845520019531, + "KL/mean": -104.88301086425781, + "KL/rejected_KL_mean": -136.42755126953125, + "KL/std": 80.35075378417969, + "epoch": 0.2393538913362702, + "fcm_dpo/beta": 0.007712847553193569, + "fcm_dpo/delta": -0.0887412428855896, + "fcm_dpo/margin": 63.08909606933594, + "fcm_dpo/q_t": 0.3963692784309387, + "grad_norm": 25.23561668395996, + "learning_rate": 4.720482655449212e-07, + "logits/chosen": -0.3967311382293701, + "logits/rejected": -0.37458351254463196, + "logps/chosen": -128.36007690429688, + "logps/ref_chosen": -55.021629333496094, + "logps/ref_rejected": -75.418212890625, + "logps/rejected": -211.84576416015625, + "loss": 1.0769, + "margin_dpo/margin_mean": 63.08909606933594, + "margin_dpo/margin_std": 95.95596313476562, + "step": 163 + }, + { + "KL/chosen_KL_mean": -66.4931411743164, + "KL/mean": -104.46180725097656, + "KL/rejected_KL_mean": -142.43045043945312, + "KL/std": 79.09434509277344, + "epoch": 0.24082232011747431, + "fcm_dpo/beta": 0.007562342099845409, + "fcm_dpo/delta": -0.18068404495716095, + "fcm_dpo/margin": 75.93731689453125, + "fcm_dpo/q_t": 0.3738352656364441, + "grad_norm": 23.940841674804688, + "learning_rate": 4.714556901942599e-07, + "logits/chosen": -0.3866020441055298, + "logits/rejected": -0.3667601943016052, + "logps/chosen": -122.1338119506836, + "logps/ref_chosen": -55.64066696166992, + "logps/ref_rejected": -79.66463470458984, + "logps/rejected": -222.0950927734375, + "loss": 0.9952, + "margin_dpo/margin_mean": 75.93731689453125, + "margin_dpo/margin_std": 90.33500671386719, + "step": 164 + }, + { + "KL/chosen_KL_mean": -70.58739471435547, + "KL/mean": -95.22163391113281, + "KL/rejected_KL_mean": -119.85586547851562, + "KL/std": 65.43978881835938, + "epoch": 0.2422907488986784, + "fcm_dpo/beta": 0.007562550250440836, + "fcm_dpo/delta": 0.027895405888557434, + "fcm_dpo/margin": 49.26846694946289, + "fcm_dpo/q_t": 0.41699641942977905, + "grad_norm": 28.192373275756836, + "learning_rate": 4.708572792802069e-07, + "logits/chosen": -0.4508388042449951, + "logits/rejected": -0.41956329345703125, + "logps/chosen": -131.89808654785156, + "logps/ref_chosen": -61.310691833496094, + "logps/ref_rejected": -73.67060852050781, + "logps/rejected": -193.52647399902344, + "loss": 1.1451, + "margin_dpo/margin_mean": 49.26846694946289, + "margin_dpo/margin_std": 85.33627319335938, + "step": 165 + }, + { + "KL/chosen_KL_mean": -62.29485321044922, + "KL/mean": -107.13589477539062, + "KL/rejected_KL_mean": -151.9769287109375, + "KL/std": 95.33642578125, + "epoch": 0.24375917767988253, + "fcm_dpo/beta": 0.007406267803162336, + "fcm_dpo/delta": -0.27418437600135803, + "fcm_dpo/margin": 89.68207550048828, + "fcm_dpo/q_t": 0.3678908348083496, + "grad_norm": 19.755449295043945, + "learning_rate": 4.702530485714461e-07, + "logits/chosen": -0.38571715354919434, + "logits/rejected": -0.395542174577713, + "logps/chosen": -113.27845764160156, + "logps/ref_chosen": -50.98360061645508, + "logps/ref_rejected": -98.09512329101562, + "logps/rejected": -250.07205200195312, + "loss": 0.9781, + "margin_dpo/margin_mean": 89.68207550048828, + "margin_dpo/margin_std": 119.04489135742188, + "step": 166 + }, + { + "KL/chosen_KL_mean": -62.64759826660156, + "KL/mean": -109.76869201660156, + "KL/rejected_KL_mean": -156.8897705078125, + "KL/std": 87.04228210449219, + "epoch": 0.24522760646108663, + "fcm_dpo/beta": 0.007213749922811985, + "fcm_dpo/delta": -0.2897433936595917, + "fcm_dpo/margin": 94.24217987060547, + "fcm_dpo/q_t": 0.3522365093231201, + "grad_norm": 21.15924072265625, + "learning_rate": 4.6964301399001877e-07, + "logits/chosen": -0.3807927370071411, + "logits/rejected": -0.3821834325790405, + "logps/chosen": -113.07169342041016, + "logps/ref_chosen": -50.424095153808594, + "logps/ref_rejected": -96.03042602539062, + "logps/rejected": -252.92019653320312, + "loss": 0.9275, + "margin_dpo/margin_mean": 94.24217224121094, + "margin_dpo/margin_std": 99.01016235351562, + "step": 167 + }, + { + "KL/chosen_KL_mean": -68.97935485839844, + "KL/mean": -104.1439208984375, + "KL/rejected_KL_mean": -139.30848693847656, + "KL/std": 81.96823120117188, + "epoch": 0.24669603524229075, + "fcm_dpo/beta": 0.007100098766386509, + "fcm_dpo/delta": -0.10204099118709564, + "fcm_dpo/margin": 70.32914733886719, + "fcm_dpo/q_t": 0.3902486562728882, + "grad_norm": 21.94905662536621, + "learning_rate": 4.690271916109034e-07, + "logits/chosen": -0.405794233083725, + "logits/rejected": -0.39295631647109985, + "logps/chosen": -118.44217681884766, + "logps/ref_chosen": -49.462825775146484, + "logps/ref_rejected": -75.30855560302734, + "logps/rejected": -214.61705017089844, + "loss": 1.0391, + "margin_dpo/margin_mean": 70.32914733886719, + "margin_dpo/margin_std": 90.95465850830078, + "step": 168 + }, + { + "KL/chosen_KL_mean": -69.83270263671875, + "KL/mean": -100.67288208007812, + "KL/rejected_KL_mean": -131.5130615234375, + "KL/std": 81.13442993164062, + "epoch": 0.24816446402349487, + "fcm_dpo/beta": 0.007038552314043045, + "fcm_dpo/delta": -0.03588036447763443, + "fcm_dpo/margin": 61.68037414550781, + "fcm_dpo/q_t": 0.4094877243041992, + "grad_norm": 21.894031524658203, + "learning_rate": 4.6840559766159235e-07, + "logits/chosen": -0.42251360416412354, + "logits/rejected": -0.403925359249115, + "logps/chosen": -129.63613891601562, + "logps/ref_chosen": -59.803443908691406, + "logps/ref_rejected": -83.34574890136719, + "logps/rejected": -214.85882568359375, + "loss": 1.1309, + "margin_dpo/margin_mean": 61.68037414550781, + "margin_dpo/margin_std": 110.4356460571289, + "step": 169 + }, + { + "KL/chosen_KL_mean": -62.970611572265625, + "KL/mean": -99.88447570800781, + "KL/rejected_KL_mean": -136.79833984375, + "KL/std": 76.65866088867188, + "epoch": 0.24963289280469897, + "fcm_dpo/beta": 0.0069792973808944225, + "fcm_dpo/delta": -0.11909201741218567, + "fcm_dpo/margin": 73.82772827148438, + "fcm_dpo/q_t": 0.3844184875488281, + "grad_norm": 18.95234489440918, + "learning_rate": 4.6777824852166437e-07, + "logits/chosen": -0.3876168131828308, + "logits/rejected": -0.37476158142089844, + "logps/chosen": -112.4423828125, + "logps/ref_chosen": -49.471771240234375, + "logps/ref_rejected": -75.91734313964844, + "logps/rejected": -212.71568298339844, + "loss": 1.0306, + "margin_dpo/margin_mean": 73.82772827148438, + "margin_dpo/margin_std": 91.29777526855469, + "step": 170 + }, + { + "KL/chosen_KL_mean": -92.04759979248047, + "KL/mean": -121.734619140625, + "KL/rejected_KL_mean": -151.42166137695312, + "KL/std": 87.79399108886719, + "epoch": 0.2511013215859031, + "fcm_dpo/beta": 0.006971217226237059, + "fcm_dpo/delta": -0.01420888677239418, + "fcm_dpo/margin": 59.374053955078125, + "fcm_dpo/q_t": 0.41368788480758667, + "grad_norm": 30.997112274169922, + "learning_rate": 4.6714516072235273e-07, + "logits/chosen": -0.3963302969932556, + "logits/rejected": -0.3728923797607422, + "logps/chosen": -176.54690551757812, + "logps/ref_chosen": -84.49931335449219, + "logps/ref_rejected": -109.38209533691406, + "logps/rejected": -260.80377197265625, + "loss": 1.1435, + "margin_dpo/margin_mean": 59.374053955078125, + "margin_dpo/margin_std": 111.98719787597656, + "step": 171 + }, + { + "KL/chosen_KL_mean": -83.34523010253906, + "KL/mean": -114.48237609863281, + "KL/rejected_KL_mean": -145.6195068359375, + "KL/std": 86.62696838378906, + "epoch": 0.2525697503671072, + "fcm_dpo/beta": 0.00694639515131712, + "fcm_dpo/delta": -0.03337009996175766, + "fcm_dpo/margin": 62.2743034362793, + "fcm_dpo/q_t": 0.4046783447265625, + "grad_norm": 21.863208770751953, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": -0.44137507677078247, + "logits/rejected": -0.4202975630760193, + "logps/chosen": -151.9991455078125, + "logps/ref_chosen": -68.65391540527344, + "logps/ref_rejected": -85.43667602539062, + "logps/rejected": -231.05618286132812, + "loss": 1.1028, + "margin_dpo/margin_mean": 62.2743034362793, + "margin_dpo/margin_std": 97.67938232421875, + "step": 172 + }, + { + "KL/chosen_KL_mean": -75.67850494384766, + "KL/mean": -107.38485717773438, + "KL/rejected_KL_mean": -139.09121704101562, + "KL/std": 80.97750854492188, + "epoch": 0.2540381791483113, + "fcm_dpo/beta": 0.006938562728464603, + "fcm_dpo/delta": -0.04095254838466644, + "fcm_dpo/margin": 63.4127082824707, + "fcm_dpo/q_t": 0.40140169858932495, + "grad_norm": 22.291126251220703, + "learning_rate": 4.6586183602616687e-07, + "logits/chosen": -0.4557979702949524, + "logits/rejected": -0.4270375072956085, + "logps/chosen": -138.72938537597656, + "logps/ref_chosen": -63.050880432128906, + "logps/ref_rejected": -78.68392181396484, + "logps/rejected": -217.775146484375, + "loss": 1.0729, + "margin_dpo/margin_mean": 63.41270446777344, + "margin_dpo/margin_std": 85.16667175292969, + "step": 173 + }, + { + "KL/chosen_KL_mean": -73.96533203125, + "KL/mean": -110.90480041503906, + "KL/rejected_KL_mean": -147.84425354003906, + "KL/std": 88.3280258178711, + "epoch": 0.2555066079295154, + "fcm_dpo/beta": 0.006883748807013035, + "fcm_dpo/delta": -0.1114603653550148, + "fcm_dpo/margin": 73.87892150878906, + "fcm_dpo/q_t": 0.3911077380180359, + "grad_norm": 23.334630966186523, + "learning_rate": 4.652116329460919e-07, + "logits/chosen": -0.3489905595779419, + "logits/rejected": -0.3669482469558716, + "logps/chosen": -127.32830810546875, + "logps/ref_chosen": -53.36296844482422, + "logps/ref_rejected": -101.91120910644531, + "logps/rejected": -249.75546264648438, + "loss": 1.0572, + "margin_dpo/margin_mean": 73.87892150878906, + "margin_dpo/margin_std": 104.1011962890625, + "step": 174 + }, + { + "KL/chosen_KL_mean": -69.70106506347656, + "KL/mean": -120.88041687011719, + "KL/rejected_KL_mean": -172.05975341796875, + "KL/std": 95.63670349121094, + "epoch": 0.25697503671071953, + "fcm_dpo/beta": 0.00668959878385067, + "fcm_dpo/delta": -0.29501020908355713, + "fcm_dpo/margin": 102.35872650146484, + "fcm_dpo/q_t": 0.34925341606140137, + "grad_norm": 25.57117462158203, + "learning_rate": 4.645557588393406e-07, + "logits/chosen": -0.3187200427055359, + "logits/rejected": -0.30461329221725464, + "logps/chosen": -115.11882019042969, + "logps/ref_chosen": -45.417762756347656, + "logps/ref_rejected": -89.50579833984375, + "logps/rejected": -261.5655517578125, + "loss": 0.9085, + "margin_dpo/margin_mean": 102.35871887207031, + "margin_dpo/margin_std": 99.24336242675781, + "step": 175 + }, + { + "KL/chosen_KL_mean": -75.16090393066406, + "KL/mean": -117.7568359375, + "KL/rejected_KL_mean": -160.35279846191406, + "KL/std": 92.66831970214844, + "epoch": 0.25844346549192365, + "fcm_dpo/beta": 0.0065691666677594185, + "fcm_dpo/delta": -0.16410863399505615, + "fcm_dpo/margin": 85.19189453125, + "fcm_dpo/q_t": 0.380332350730896, + "grad_norm": 22.5208683013916, + "learning_rate": 4.638942309888058e-07, + "logits/chosen": -0.3513135313987732, + "logits/rejected": -0.369649738073349, + "logps/chosen": -125.61373901367188, + "logps/ref_chosen": -50.452842712402344, + "logps/ref_rejected": -95.5589599609375, + "logps/rejected": -255.91177368164062, + "loss": 1.0084, + "margin_dpo/margin_mean": 85.19189453125, + "margin_dpo/margin_std": 107.12543487548828, + "step": 176 + }, + { + "KL/chosen_KL_mean": -87.45088195800781, + "KL/mean": -129.8768310546875, + "KL/rejected_KL_mean": -172.30276489257812, + "KL/std": 101.22750854492188, + "epoch": 0.2599118942731278, + "fcm_dpo/beta": 0.006470865570008755, + "fcm_dpo/delta": -0.15313297510147095, + "fcm_dpo/margin": 84.85188293457031, + "fcm_dpo/q_t": 0.3814983367919922, + "grad_norm": 33.10342025756836, + "learning_rate": 4.6322706682636137e-07, + "logits/chosen": -0.38505709171295166, + "logits/rejected": -0.37769830226898193, + "logps/chosen": -148.66734313964844, + "logps/ref_chosen": -61.216468811035156, + "logps/ref_rejected": -95.89378356933594, + "logps/rejected": -268.196533203125, + "loss": 1.0131, + "margin_dpo/margin_mean": 84.85188293457031, + "margin_dpo/margin_std": 107.24163818359375, + "step": 177 + }, + { + "KL/chosen_KL_mean": -96.93707275390625, + "KL/mean": -150.7945098876953, + "KL/rejected_KL_mean": -204.65194702148438, + "KL/std": 121.38895416259766, + "epoch": 0.26138032305433184, + "fcm_dpo/beta": 0.006285130977630615, + "fcm_dpo/delta": -0.28755688667297363, + "fcm_dpo/margin": 107.7148666381836, + "fcm_dpo/q_t": 0.36004963517189026, + "grad_norm": 27.839982986450195, + "learning_rate": 4.6255428393240354e-07, + "logits/chosen": -0.27259236574172974, + "logits/rejected": -0.26497703790664673, + "logps/chosen": -155.20187377929688, + "logps/ref_chosen": -58.26478958129883, + "logps/ref_rejected": -105.3653335571289, + "logps/rejected": -310.01727294921875, + "loss": 0.9589, + "margin_dpo/margin_mean": 107.7148666381836, + "margin_dpo/margin_std": 132.7176971435547, + "step": 178 + }, + { + "KL/chosen_KL_mean": -97.64852905273438, + "KL/mean": -136.50204467773438, + "KL/rejected_KL_mean": -175.3555908203125, + "KL/std": 101.76957702636719, + "epoch": 0.26284875183553597, + "fcm_dpo/beta": 0.006197200156748295, + "fcm_dpo/delta": -0.08410830795764923, + "fcm_dpo/margin": 77.70704650878906, + "fcm_dpo/q_t": 0.39651763439178467, + "grad_norm": 37.91785430908203, + "learning_rate": 4.6187590003538724e-07, + "logits/chosen": -0.29223281145095825, + "logits/rejected": -0.29845482110977173, + "logps/chosen": -158.70684814453125, + "logps/ref_chosen": -61.05832290649414, + "logps/ref_rejected": -90.52782440185547, + "logps/rejected": -265.8834228515625, + "loss": 1.0988, + "margin_dpo/margin_mean": 77.70704650878906, + "margin_dpo/margin_std": 125.48446655273438, + "step": 179 + }, + { + "KL/chosen_KL_mean": -83.23408508300781, + "KL/mean": -131.74658203125, + "KL/rejected_KL_mean": -180.2590789794922, + "KL/std": 91.78398132324219, + "epoch": 0.2643171806167401, + "fcm_dpo/beta": 0.006104484666138887, + "fcm_dpo/delta": -0.19836077094078064, + "fcm_dpo/margin": 97.02499389648438, + "fcm_dpo/q_t": 0.3690943717956543, + "grad_norm": 21.05206871032715, + "learning_rate": 4.611919330113591e-07, + "logits/chosen": -0.2990095019340515, + "logits/rejected": -0.29073792695999146, + "logps/chosen": -137.57681274414062, + "logps/ref_chosen": -54.34272003173828, + "logps/ref_rejected": -98.21183776855469, + "logps/rejected": -278.4709167480469, + "loss": 0.9838, + "margin_dpo/margin_mean": 97.02499389648438, + "margin_dpo/margin_std": 113.17401123046875, + "step": 180 + }, + { + "KL/chosen_KL_mean": -71.93513488769531, + "KL/mean": -102.62535095214844, + "KL/rejected_KL_mean": -133.31556701660156, + "KL/std": 82.91169738769531, + "epoch": 0.2657856093979442, + "fcm_dpo/beta": 0.006095272954553366, + "fcm_dpo/delta": 0.02632547914981842, + "fcm_dpo/margin": 61.38043212890625, + "fcm_dpo/q_t": 0.41590094566345215, + "grad_norm": 20.209400177001953, + "learning_rate": 4.605024008834863e-07, + "logits/chosen": -0.37073230743408203, + "logits/rejected": -0.34592998027801514, + "logps/chosen": -126.93559265136719, + "logps/ref_chosen": -55.000457763671875, + "logps/ref_rejected": -61.656166076660156, + "logps/rejected": -194.97174072265625, + "loss": 1.1374, + "margin_dpo/margin_mean": 61.38043212890625, + "margin_dpo/margin_std": 102.93232727050781, + "step": 181 + }, + { + "KL/chosen_KL_mean": -68.23200225830078, + "KL/mean": -119.69134521484375, + "KL/rejected_KL_mean": -171.15066528320312, + "KL/std": 100.23439025878906, + "epoch": 0.26725403817914833, + "fcm_dpo/beta": 0.005980286747217178, + "fcm_dpo/delta": -0.2230871319770813, + "fcm_dpo/margin": 102.91866302490234, + "fcm_dpo/q_t": 0.3642627000808716, + "grad_norm": 20.144750595092773, + "learning_rate": 4.598073218215817e-07, + "logits/chosen": -0.31923383474349976, + "logits/rejected": -0.32709991931915283, + "logps/chosen": -109.33985900878906, + "logps/ref_chosen": -41.107852935791016, + "logps/ref_rejected": -89.5215835571289, + "logps/rejected": -260.6722412109375, + "loss": 0.9789, + "margin_dpo/margin_mean": 102.91866302490234, + "margin_dpo/margin_std": 120.07926940917969, + "step": 182 + }, + { + "KL/chosen_KL_mean": -99.28335571289062, + "KL/mean": -128.1656036376953, + "KL/rejected_KL_mean": -157.0478515625, + "KL/std": 81.12559509277344, + "epoch": 0.2687224669603524, + "fcm_dpo/beta": 0.00591336889192462, + "fcm_dpo/delta": -0.07965878397226334, + "fcm_dpo/margin": 57.76450729370117, + "fcm_dpo/q_t": 0.42185482382774353, + "grad_norm": 21.069660186767578, + "learning_rate": 4.5910671414162484e-07, + "logits/chosen": -0.30978119373321533, + "logits/rejected": -0.29306042194366455, + "logps/chosen": -156.80792236328125, + "logps/ref_chosen": -57.52456283569336, + "logps/ref_rejected": -75.97572326660156, + "logps/rejected": -233.02359008789062, + "loss": 1.1494, + "margin_dpo/margin_mean": 57.764503479003906, + "margin_dpo/margin_std": 89.65037536621094, + "step": 183 + }, + { + "KL/chosen_KL_mean": -80.81704711914062, + "KL/mean": -110.47319030761719, + "KL/rejected_KL_mean": -140.12933349609375, + "KL/std": 76.08272552490234, + "epoch": 0.2701908957415565, + "fcm_dpo/beta": 0.005951171740889549, + "fcm_dpo/delta": 0.04741087928414345, + "fcm_dpo/margin": 59.312294006347656, + "fcm_dpo/q_t": 0.4209594130516052, + "grad_norm": 21.374256134033203, + "learning_rate": 4.5840059630527985e-07, + "logits/chosen": -0.41061311960220337, + "logits/rejected": -0.4004828929901123, + "logps/chosen": -139.36199951171875, + "logps/ref_chosen": -58.544952392578125, + "logps/ref_rejected": -76.63406372070312, + "logps/rejected": -216.76339721679688, + "loss": 1.1399, + "margin_dpo/margin_mean": 59.312294006347656, + "margin_dpo/margin_std": 94.55437469482422, + "step": 184 + }, + { + "KL/chosen_KL_mean": -88.34465026855469, + "KL/mean": -112.53829193115234, + "KL/rejected_KL_mean": -136.73193359375, + "KL/std": 88.54534912109375, + "epoch": 0.27165932452276065, + "fcm_dpo/beta": 0.005989473778754473, + "fcm_dpo/delta": 0.1118379533290863, + "fcm_dpo/margin": 48.38727951049805, + "fcm_dpo/q_t": 0.4377959966659546, + "grad_norm": 20.70436668395996, + "learning_rate": 4.5768898691940836e-07, + "logits/chosen": -0.3213365077972412, + "logits/rejected": -0.2884293794631958, + "logps/chosen": -150.37049865722656, + "logps/ref_chosen": -62.025848388671875, + "logps/ref_rejected": -73.7625961303711, + "logps/rejected": -210.49453735351562, + "loss": 1.2074, + "margin_dpo/margin_mean": 48.38727569580078, + "margin_dpo/margin_std": 105.9352798461914, + "step": 185 + }, + { + "KL/chosen_KL_mean": -81.42449188232422, + "KL/mean": -126.01622009277344, + "KL/rejected_KL_mean": -170.60794067382812, + "KL/std": 91.03404235839844, + "epoch": 0.27312775330396477, + "fcm_dpo/beta": 0.005945336539298296, + "fcm_dpo/delta": -0.13385680317878723, + "fcm_dpo/margin": 89.18345642089844, + "fcm_dpo/q_t": 0.3814866244792938, + "grad_norm": 34.85935974121094, + "learning_rate": 4.5697190473557947e-07, + "logits/chosen": -0.41816607117652893, + "logits/rejected": -0.3946566581726074, + "logps/chosen": -150.7779541015625, + "logps/ref_chosen": -69.35346984863281, + "logps/ref_rejected": -88.07244873046875, + "logps/rejected": -258.6803894042969, + "loss": 1.0069, + "margin_dpo/margin_mean": 89.18345642089844, + "margin_dpo/margin_std": 102.38252258300781, + "step": 186 + }, + { + "KL/chosen_KL_mean": -77.34542846679688, + "KL/mean": -114.29693603515625, + "KL/rejected_KL_mean": -151.24844360351562, + "KL/std": 86.13810729980469, + "epoch": 0.2745961820851689, + "fcm_dpo/beta": 0.005925598554313183, + "fcm_dpo/delta": -0.038912609219551086, + "fcm_dpo/margin": 73.90303039550781, + "fcm_dpo/q_t": 0.4008300006389618, + "grad_norm": 29.05039405822754, + "learning_rate": 4.5624936864957555e-07, + "logits/chosen": -0.3810155987739563, + "logits/rejected": -0.37561988830566406, + "logps/chosen": -130.1018829345703, + "logps/ref_chosen": -52.7564582824707, + "logps/ref_rejected": -81.96910095214844, + "logps/rejected": -233.21754455566406, + "loss": 1.0696, + "margin_dpo/margin_mean": 73.90303039550781, + "margin_dpo/margin_std": 94.80323791503906, + "step": 187 + }, + { + "KL/chosen_KL_mean": -73.07807922363281, + "KL/mean": -118.4848403930664, + "KL/rejected_KL_mean": -163.8916015625, + "KL/std": 96.97216796875, + "epoch": 0.27606461086637296, + "fcm_dpo/beta": 0.005849046632647514, + "fcm_dpo/delta": -0.13469749689102173, + "fcm_dpo/margin": 90.81352233886719, + "fcm_dpo/q_t": 0.3821578025817871, + "grad_norm": 35.57075881958008, + "learning_rate": 4.5552139770089454e-07, + "logits/chosen": -0.35656416416168213, + "logits/rejected": -0.36152058839797974, + "logps/chosen": -122.49356842041016, + "logps/ref_chosen": -49.415489196777344, + "logps/ref_rejected": -89.54043579101562, + "logps/rejected": -253.43203735351562, + "loss": 1.0084, + "margin_dpo/margin_mean": 90.81352233886719, + "margin_dpo/margin_std": 105.5975341796875, + "step": 188 + }, + { + "KL/chosen_KL_mean": -88.35533142089844, + "KL/mean": -125.64856719970703, + "KL/rejected_KL_mean": -162.94180297851562, + "KL/std": 98.73692321777344, + "epoch": 0.2775330396475771, + "fcm_dpo/beta": 0.005813870579004288, + "fcm_dpo/delta": -0.0343741774559021, + "fcm_dpo/margin": 74.58646392822266, + "fcm_dpo/q_t": 0.4062124490737915, + "grad_norm": 29.83676528930664, + "learning_rate": 4.5478801107224794e-07, + "logits/chosen": -0.3425024151802063, + "logits/rejected": -0.32220566272735596, + "logps/chosen": -140.75428771972656, + "logps/ref_chosen": -52.39896011352539, + "logps/ref_rejected": -72.16735076904297, + "logps/rejected": -235.10916137695312, + "loss": 1.1033, + "margin_dpo/margin_mean": 74.58646392822266, + "margin_dpo/margin_std": 120.13783264160156, + "step": 189 + }, + { + "KL/chosen_KL_mean": -93.82635498046875, + "KL/mean": -138.6332550048828, + "KL/rejected_KL_mean": -183.44015502929688, + "KL/std": 105.86849975585938, + "epoch": 0.2790014684287812, + "fcm_dpo/beta": 0.0057814717292785645, + "fcm_dpo/delta": -0.12157705426216125, + "fcm_dpo/margin": 89.61378479003906, + "fcm_dpo/q_t": 0.3877827227115631, + "grad_norm": 21.399682998657227, + "learning_rate": 4.5404922808905543e-07, + "logits/chosen": -0.3890034854412079, + "logits/rejected": -0.37827688455581665, + "logps/chosen": -158.50942993164062, + "logps/ref_chosen": -64.68305969238281, + "logps/ref_rejected": -102.55052185058594, + "logps/rejected": -285.99066162109375, + "loss": 1.0533, + "margin_dpo/margin_mean": 89.61378479003906, + "margin_dpo/margin_std": 122.82894897460938, + "step": 190 + }, + { + "KL/chosen_KL_mean": -86.75233459472656, + "KL/mean": -150.7803955078125, + "KL/rejected_KL_mean": -214.80844116210938, + "KL/std": 124.46101379394531, + "epoch": 0.28046989720998533, + "fcm_dpo/beta": 0.005586773157119751, + "fcm_dpo/delta": -0.32792675495147705, + "fcm_dpo/margin": 128.05612182617188, + "fcm_dpo/q_t": 0.34851568937301636, + "grad_norm": 24.331701278686523, + "learning_rate": 4.5330506821893565e-07, + "logits/chosen": -0.3571997880935669, + "logits/rejected": -0.3361099660396576, + "logps/chosen": -155.4112091064453, + "logps/ref_chosen": -68.65887451171875, + "logps/ref_rejected": -110.1396713256836, + "logps/rejected": -324.9481201171875, + "loss": 0.912, + "margin_dpo/margin_mean": 128.05612182617188, + "margin_dpo/margin_std": 137.22451782226562, + "step": 191 + }, + { + "KL/chosen_KL_mean": -114.01398468017578, + "KL/mean": -156.28001403808594, + "KL/rejected_KL_mean": -198.54605102539062, + "KL/std": 106.98095703125, + "epoch": 0.28193832599118945, + "fcm_dpo/beta": 0.005530376918613911, + "fcm_dpo/delta": -0.06909237802028656, + "fcm_dpo/margin": 84.53204345703125, + "fcm_dpo/q_t": 0.39953508973121643, + "grad_norm": 28.160974502563477, + "learning_rate": 4.5255555107119336e-07, + "logits/chosen": -0.31193774938583374, + "logits/rejected": -0.31198960542678833, + "logps/chosen": -183.74090576171875, + "logps/ref_chosen": -69.72691345214844, + "logps/ref_rejected": -103.32135009765625, + "logps/rejected": -301.86737060546875, + "loss": 1.089, + "margin_dpo/margin_mean": 84.53204345703125, + "margin_dpo/margin_std": 131.72988891601562, + "step": 192 + }, + { + "KL/chosen_KL_mean": -114.42250061035156, + "KL/mean": -138.80294799804688, + "KL/rejected_KL_mean": -163.18338012695312, + "KL/std": 97.2500228881836, + "epoch": 0.2834067547723935, + "fcm_dpo/beta": 0.005518913269042969, + "fcm_dpo/delta": 0.021666744723916054, + "fcm_dpo/margin": 48.76087951660156, + "fcm_dpo/q_t": 0.43647685647010803, + "grad_norm": 31.76787567138672, + "learning_rate": 4.5180069639630236e-07, + "logits/chosen": -0.3469330072402954, + "logits/rejected": -0.33776986598968506, + "logps/chosen": -174.61300659179688, + "logps/ref_chosen": -60.19049835205078, + "logps/ref_rejected": -76.40755462646484, + "logps/rejected": -239.5909423828125, + "loss": 1.2443, + "margin_dpo/margin_mean": 48.76087951660156, + "margin_dpo/margin_std": 123.41360473632812, + "step": 193 + }, + { + "KL/chosen_KL_mean": -71.24337768554688, + "KL/mean": -112.21340942382812, + "KL/rejected_KL_mean": -153.18345642089844, + "KL/std": 83.32481384277344, + "epoch": 0.28487518355359764, + "fcm_dpo/beta": 0.0054954723455011845, + "fcm_dpo/delta": -0.051623422652482986, + "fcm_dpo/margin": 81.94007873535156, + "fcm_dpo/q_t": 0.395707905292511, + "grad_norm": 20.26607894897461, + "learning_rate": 4.510405240853854e-07, + "logits/chosen": -0.2368777096271515, + "logits/rejected": -0.21953874826431274, + "logps/chosen": -109.08374786376953, + "logps/ref_chosen": -37.84037399291992, + "logps/ref_rejected": -60.684783935546875, + "logps/rejected": -213.8682403564453, + "loss": 1.0437, + "margin_dpo/margin_mean": 81.94007873535156, + "margin_dpo/margin_std": 90.14231872558594, + "step": 194 + }, + { + "KL/chosen_KL_mean": -112.94318389892578, + "KL/mean": -156.35760498046875, + "KL/rejected_KL_mean": -199.77203369140625, + "KL/std": 102.81307983398438, + "epoch": 0.28634361233480177, + "fcm_dpo/beta": 0.0054633598774671555, + "fcm_dpo/delta": -0.07621531188488007, + "fcm_dpo/margin": 86.82884216308594, + "fcm_dpo/q_t": 0.3926421105861664, + "grad_norm": 26.608348846435547, + "learning_rate": 4.5027505416968985e-07, + "logits/chosen": -0.27807819843292236, + "logits/rejected": -0.29660797119140625, + "logps/chosen": -167.83474731445312, + "logps/ref_chosen": -54.891571044921875, + "logps/ref_rejected": -96.77095794677734, + "logps/rejected": -296.54296875, + "loss": 1.0427, + "margin_dpo/margin_mean": 86.82884216308594, + "margin_dpo/margin_std": 105.82039642333984, + "step": 195 + }, + { + "KL/chosen_KL_mean": -87.1651382446289, + "KL/mean": -137.2007598876953, + "KL/rejected_KL_mean": -187.23635864257812, + "KL/std": 106.05245208740234, + "epoch": 0.2878120411160059, + "fcm_dpo/beta": 0.005381887778639793, + "fcm_dpo/delta": -0.14310941100120544, + "fcm_dpo/margin": 100.07121276855469, + "fcm_dpo/q_t": 0.3824601173400879, + "grad_norm": 20.205522537231445, + "learning_rate": 4.495043068200599e-07, + "logits/chosen": -0.3048306703567505, + "logits/rejected": -0.2887340188026428, + "logps/chosen": -140.41038513183594, + "logps/ref_chosen": -53.245243072509766, + "logps/ref_rejected": -76.05294799804688, + "logps/rejected": -263.289306640625, + "loss": 1.0251, + "margin_dpo/margin_mean": 100.07121276855469, + "margin_dpo/margin_std": 125.76506042480469, + "step": 196 + }, + { + "KL/chosen_KL_mean": -87.905029296875, + "KL/mean": -126.29278564453125, + "KL/rejected_KL_mean": -164.6805419921875, + "KL/std": 89.6479263305664, + "epoch": 0.28928046989721, + "fcm_dpo/beta": 0.005379960872232914, + "fcm_dpo/delta": -0.01339656300842762, + "fcm_dpo/margin": 76.7755126953125, + "fcm_dpo/q_t": 0.40633296966552734, + "grad_norm": 25.854190826416016, + "learning_rate": 4.4872830234640493e-07, + "logits/chosen": -0.2854430675506592, + "logits/rejected": -0.2793597877025604, + "logps/chosen": -148.3253631591797, + "logps/ref_chosen": -60.42033386230469, + "logps/ref_rejected": -77.20890808105469, + "logps/rejected": -241.8894500732422, + "loss": 1.0832, + "margin_dpo/margin_mean": 76.7755126953125, + "margin_dpo/margin_std": 100.7091064453125, + "step": 197 + }, + { + "KL/chosen_KL_mean": -97.10411071777344, + "KL/mean": -145.3472900390625, + "KL/rejected_KL_mean": -193.59048461914062, + "KL/std": 110.99395751953125, + "epoch": 0.2907488986784141, + "fcm_dpo/beta": 0.005326003301888704, + "fcm_dpo/delta": -0.11684248596429825, + "fcm_dpo/margin": 96.48635864257812, + "fcm_dpo/q_t": 0.38827261328697205, + "grad_norm": 22.704126358032227, + "learning_rate": 4.479470611971645e-07, + "logits/chosen": -0.32224997878074646, + "logits/rejected": -0.32293713092803955, + "logps/chosen": -152.14028930664062, + "logps/ref_chosen": -55.03618621826172, + "logps/ref_rejected": -97.24325561523438, + "logps/rejected": -290.833740234375, + "loss": 1.0307, + "margin_dpo/margin_mean": 96.48635864257812, + "margin_dpo/margin_std": 124.41645050048828, + "step": 198 + }, + { + "KL/chosen_KL_mean": -93.67039489746094, + "KL/mean": -140.52859497070312, + "KL/rejected_KL_mean": -187.3867950439453, + "KL/std": 99.46040344238281, + "epoch": 0.2922173274596182, + "fcm_dpo/beta": 0.0052545820362865925, + "fcm_dpo/delta": -0.09539773315191269, + "fcm_dpo/margin": 93.71640014648438, + "fcm_dpo/q_t": 0.38956761360168457, + "grad_norm": 34.478763580322266, + "learning_rate": 4.471606039587695e-07, + "logits/chosen": -0.3307437300682068, + "logits/rejected": -0.31390073895454407, + "logps/chosen": -150.4992218017578, + "logps/ref_chosen": -56.828826904296875, + "logps/ref_rejected": -84.64820861816406, + "logps/rejected": -272.0350036621094, + "loss": 1.0441, + "margin_dpo/margin_mean": 93.71640014648438, + "margin_dpo/margin_std": 118.5833969116211, + "step": 199 + }, + { + "KL/chosen_KL_mean": -91.72415161132812, + "KL/mean": -139.52906799316406, + "KL/rejected_KL_mean": -187.333984375, + "KL/std": 108.85971069335938, + "epoch": 0.2936857562408223, + "fcm_dpo/beta": 0.0052094850689172745, + "fcm_dpo/delta": -0.10084635764360428, + "fcm_dpo/margin": 95.60984802246094, + "fcm_dpo/q_t": 0.3912206292152405, + "grad_norm": 20.905105590820312, + "learning_rate": 4.4636895135509966e-07, + "logits/chosen": -0.29428964853286743, + "logits/rejected": -0.2763988971710205, + "logps/chosen": -144.79119873046875, + "logps/ref_chosen": -53.06706237792969, + "logps/ref_rejected": -80.60843658447266, + "logps/rejected": -267.9424133300781, + "loss": 1.0584, + "margin_dpo/margin_mean": 95.60984802246094, + "margin_dpo/margin_std": 135.0447998046875, + "step": 200 + }, + { + "KL/chosen_KL_mean": -94.4271469116211, + "KL/mean": -142.72763061523438, + "KL/rejected_KL_mean": -191.028076171875, + "KL/std": 113.81238555908203, + "epoch": 0.29515418502202645, + "fcm_dpo/beta": 0.005169622600078583, + "fcm_dpo/delta": -0.1018773764371872, + "fcm_dpo/margin": 96.60094451904297, + "fcm_dpo/q_t": 0.389159232378006, + "grad_norm": 26.09699249267578, + "learning_rate": 4.455721242469372e-07, + "logits/chosen": -0.34645718336105347, + "logits/rejected": -0.34164953231811523, + "logps/chosen": -169.82937622070312, + "logps/ref_chosen": -75.4022216796875, + "logps/ref_rejected": -114.80821990966797, + "logps/rejected": -305.8363037109375, + "loss": 1.0446, + "margin_dpo/margin_mean": 96.60094451904297, + "margin_dpo/margin_std": 127.17774963378906, + "step": 201 + }, + { + "KL/chosen_KL_mean": -98.94967651367188, + "KL/mean": -132.004150390625, + "KL/rejected_KL_mean": -165.05862426757812, + "KL/std": 95.95150756835938, + "epoch": 0.2966226138032305, + "fcm_dpo/beta": 0.0051773954182863235, + "fcm_dpo/delta": 0.05868521332740784, + "fcm_dpo/margin": 66.10896301269531, + "fcm_dpo/q_t": 0.42411088943481445, + "grad_norm": 20.637813568115234, + "learning_rate": 4.4477014363141755e-07, + "logits/chosen": -0.30686646699905396, + "logits/rejected": -0.31850093603134155, + "logps/chosen": -149.05099487304688, + "logps/ref_chosen": -50.101318359375, + "logps/ref_rejected": -86.98503112792969, + "logps/rejected": -252.04367065429688, + "loss": 1.1671, + "margin_dpo/margin_mean": 66.10896301269531, + "margin_dpo/margin_std": 123.44126892089844, + "step": 202 + }, + { + "KL/chosen_KL_mean": -98.62371826171875, + "KL/mean": -140.30938720703125, + "KL/rejected_KL_mean": -181.99505615234375, + "KL/std": 101.09996795654297, + "epoch": 0.29809104258443464, + "fcm_dpo/beta": 0.005167345050722361, + "fcm_dpo/delta": -0.03151167184114456, + "fcm_dpo/margin": 83.371337890625, + "fcm_dpo/q_t": 0.4019864797592163, + "grad_norm": 25.103071212768555, + "learning_rate": 4.439630306414758e-07, + "logits/chosen": -0.31344640254974365, + "logits/rejected": -0.30354416370391846, + "logps/chosen": -159.23341369628906, + "logps/ref_chosen": -60.60969543457031, + "logps/ref_rejected": -85.89596557617188, + "logps/rejected": -267.8910217285156, + "loss": 1.0711, + "margin_dpo/margin_mean": 83.371337890625, + "margin_dpo/margin_std": 107.46673583984375, + "step": 203 + }, + { + "KL/chosen_KL_mean": -110.24484252929688, + "KL/mean": -148.1464385986328, + "KL/rejected_KL_mean": -186.04803466796875, + "KL/std": 108.95531463623047, + "epoch": 0.29955947136563876, + "fcm_dpo/beta": 0.005168107338249683, + "fcm_dpo/delta": 0.008405912667512894, + "fcm_dpo/margin": 75.80317687988281, + "fcm_dpo/q_t": 0.41412895917892456, + "grad_norm": 25.396324157714844, + "learning_rate": 4.431508065452897e-07, + "logits/chosen": -0.40366190671920776, + "logits/rejected": -0.3658892512321472, + "logps/chosen": -190.40980529785156, + "logps/ref_chosen": -80.16496276855469, + "logps/ref_rejected": -87.69590759277344, + "logps/rejected": -273.74395751953125, + "loss": 1.1248, + "margin_dpo/margin_mean": 75.80317687988281, + "margin_dpo/margin_std": 123.09947204589844, + "step": 204 + }, + { + "KL/chosen_KL_mean": -108.8935775756836, + "KL/mean": -159.8162841796875, + "KL/rejected_KL_mean": -210.739013671875, + "KL/std": 114.94783020019531, + "epoch": 0.3010279001468429, + "fcm_dpo/beta": 0.005101551301777363, + "fcm_dpo/delta": -0.12394419312477112, + "fcm_dpo/margin": 101.84541320800781, + "fcm_dpo/q_t": 0.38280677795410156, + "grad_norm": 25.676708221435547, + "learning_rate": 4.4233349274571974e-07, + "logits/chosen": -0.2786010801792145, + "logits/rejected": -0.2449745088815689, + "logps/chosen": -168.2783203125, + "logps/ref_chosen": -59.384735107421875, + "logps/ref_rejected": -85.12505340576172, + "logps/rejected": -295.86407470703125, + "loss": 1.0303, + "margin_dpo/margin_mean": 101.84541320800781, + "margin_dpo/margin_std": 124.0643310546875, + "step": 205 + }, + { + "KL/chosen_KL_mean": -103.83800506591797, + "KL/mean": -158.62493896484375, + "KL/rejected_KL_mean": -213.41188049316406, + "KL/std": 111.25010681152344, + "epoch": 0.302496328928047, + "fcm_dpo/beta": 0.005038215778768063, + "fcm_dpo/delta": -0.15676981210708618, + "fcm_dpo/margin": 109.57386779785156, + "fcm_dpo/q_t": 0.374674528837204, + "grad_norm": 31.18449592590332, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": -0.2506366968154907, + "logits/rejected": -0.25460052490234375, + "logps/chosen": -150.80250549316406, + "logps/ref_chosen": -46.964500427246094, + "logps/ref_rejected": -98.9534912109375, + "logps/rejected": -312.3653564453125, + "loss": 0.9924, + "margin_dpo/margin_mean": 109.57386016845703, + "margin_dpo/margin_std": 120.0478515625, + "step": 206 + }, + { + "KL/chosen_KL_mean": -93.25586700439453, + "KL/mean": -158.32984924316406, + "KL/rejected_KL_mean": -223.40382385253906, + "KL/std": 127.83676147460938, + "epoch": 0.3039647577092511, + "fcm_dpo/beta": 0.004938926547765732, + "fcm_dpo/delta": -0.25064384937286377, + "fcm_dpo/margin": 130.14794921875, + "fcm_dpo/q_t": 0.36273083090782166, + "grad_norm": 25.51757049560547, + "learning_rate": 4.4068368231789365e-07, + "logits/chosen": -0.3203808069229126, + "logits/rejected": -0.29214242100715637, + "logps/chosen": -149.3121337890625, + "logps/ref_chosen": -56.05625915527344, + "logps/ref_rejected": -84.44779968261719, + "logps/rejected": -307.85162353515625, + "loss": 0.955, + "margin_dpo/margin_mean": 130.14796447753906, + "margin_dpo/margin_std": 148.46749877929688, + "step": 207 + }, + { + "KL/chosen_KL_mean": -143.90771484375, + "KL/mean": -192.93313598632812, + "KL/rejected_KL_mean": -241.9585418701172, + "KL/std": 121.01240539550781, + "epoch": 0.3054331864904552, + "fcm_dpo/beta": 0.004868675023317337, + "fcm_dpo/delta": -0.07939005643129349, + "fcm_dpo/margin": 98.05082702636719, + "fcm_dpo/q_t": 0.39406818151474, + "grad_norm": 27.979568481445312, + "learning_rate": 4.398512291636768e-07, + "logits/chosen": -0.35994622111320496, + "logits/rejected": -0.3440352976322174, + "logps/chosen": -210.975341796875, + "logps/ref_chosen": -67.06761169433594, + "logps/ref_rejected": -94.28689575195312, + "logps/rejected": -336.24542236328125, + "loss": 1.08, + "margin_dpo/margin_mean": 98.05082702636719, + "margin_dpo/margin_std": 146.68267822265625, + "step": 208 + }, + { + "KL/chosen_KL_mean": -118.966064453125, + "KL/mean": -162.17567443847656, + "KL/rejected_KL_mean": -205.38528442382812, + "KL/std": 105.17567443847656, + "epoch": 0.3069016152716593, + "fcm_dpo/beta": 0.004856686107814312, + "fcm_dpo/delta": -0.02013271115720272, + "fcm_dpo/margin": 86.41921997070312, + "fcm_dpo/q_t": 0.40564462542533875, + "grad_norm": 29.448095321655273, + "learning_rate": 4.3901377325300857e-07, + "logits/chosen": -0.24659614264965057, + "logits/rejected": -0.23566904664039612, + "logps/chosen": -175.14776611328125, + "logps/ref_chosen": -56.18169403076172, + "logps/ref_rejected": -80.94152069091797, + "logps/rejected": -286.3267822265625, + "loss": 1.1089, + "margin_dpo/margin_mean": 86.41921997070312, + "margin_dpo/margin_std": 132.27151489257812, + "step": 209 + }, + { + "KL/chosen_KL_mean": -103.5418472290039, + "KL/mean": -154.00094604492188, + "KL/rejected_KL_mean": -204.46005249023438, + "KL/std": 109.31126403808594, + "epoch": 0.30837004405286345, + "fcm_dpo/beta": 0.004831655882298946, + "fcm_dpo/delta": -0.0897798091173172, + "fcm_dpo/margin": 100.91820526123047, + "fcm_dpo/q_t": 0.3923318386077881, + "grad_norm": 25.811580657958984, + "learning_rate": 4.381713366536311e-07, + "logits/chosen": -0.2856072783470154, + "logits/rejected": -0.2767780125141144, + "logps/chosen": -149.91366577148438, + "logps/ref_chosen": -46.371822357177734, + "logps/ref_rejected": -76.68162536621094, + "logps/rejected": -281.14166259765625, + "loss": 1.0508, + "margin_dpo/margin_mean": 100.91820526123047, + "margin_dpo/margin_std": 132.74383544921875, + "step": 210 + }, + { + "KL/chosen_KL_mean": -145.32662963867188, + "KL/mean": -188.92605590820312, + "KL/rejected_KL_mean": -232.5254669189453, + "KL/std": 120.26460266113281, + "epoch": 0.30983847283406757, + "fcm_dpo/beta": 0.004806230776011944, + "fcm_dpo/delta": -0.019507823511958122, + "fcm_dpo/margin": 87.1988296508789, + "fcm_dpo/q_t": 0.40984851121902466, + "grad_norm": 27.256078720092773, + "learning_rate": 4.373239415645323e-07, + "logits/chosen": -0.35860946774482727, + "logits/rejected": -0.31967782974243164, + "logps/chosen": -224.25897216796875, + "logps/ref_chosen": -78.93235778808594, + "logps/ref_rejected": -86.82098388671875, + "logps/rejected": -319.346435546875, + "loss": 1.1129, + "margin_dpo/margin_mean": 87.19883728027344, + "margin_dpo/margin_std": 139.08685302734375, + "step": 211 + }, + { + "KL/chosen_KL_mean": -117.86914825439453, + "KL/mean": -179.65806579589844, + "KL/rejected_KL_mean": -241.44700622558594, + "KL/std": 132.9044952392578, + "epoch": 0.31130690161527164, + "fcm_dpo/beta": 0.004715276416391134, + "fcm_dpo/delta": -0.18955346941947937, + "fcm_dpo/margin": 123.57785034179688, + "fcm_dpo/q_t": 0.372633695602417, + "grad_norm": 25.3454647064209, + "learning_rate": 4.3647161031536086e-07, + "logits/chosen": -0.34758710861206055, + "logits/rejected": -0.33722245693206787, + "logps/chosen": -176.066162109375, + "logps/ref_chosen": -58.19701385498047, + "logps/ref_rejected": -103.05785369873047, + "logps/rejected": -344.5048522949219, + "loss": 1.0062, + "margin_dpo/margin_mean": 123.57785034179688, + "margin_dpo/margin_std": 148.7064208984375, + "step": 212 + }, + { + "KL/chosen_KL_mean": -108.09127807617188, + "KL/mean": -165.18792724609375, + "KL/rejected_KL_mean": -222.28460693359375, + "KL/std": 112.09000396728516, + "epoch": 0.31277533039647576, + "fcm_dpo/beta": 0.004654336720705032, + "fcm_dpo/delta": -0.13551779091358185, + "fcm_dpo/margin": 114.19332885742188, + "fcm_dpo/q_t": 0.38082438707351685, + "grad_norm": 22.01609992980957, + "learning_rate": 4.3561436536583774e-07, + "logits/chosen": -0.3316152095794678, + "logits/rejected": -0.3008173108100891, + "logps/chosen": -175.60398864746094, + "logps/ref_chosen": -67.51271057128906, + "logps/ref_rejected": -93.91471862792969, + "logps/rejected": -316.1993408203125, + "loss": 1.0141, + "margin_dpo/margin_mean": 114.19332885742188, + "margin_dpo/margin_std": 134.8514404296875, + "step": 213 + }, + { + "KL/chosen_KL_mean": -97.71200561523438, + "KL/mean": -148.03985595703125, + "KL/rejected_KL_mean": -198.36767578125, + "KL/std": 111.36173248291016, + "epoch": 0.3142437591776799, + "fcm_dpo/beta": 0.004625334870070219, + "fcm_dpo/delta": -0.0671173483133316, + "fcm_dpo/margin": 100.65567016601562, + "fcm_dpo/q_t": 0.3962777256965637, + "grad_norm": 21.930564880371094, + "learning_rate": 4.3475222930516473e-07, + "logits/chosen": -0.2828127145767212, + "logits/rejected": -0.2885119915008545, + "logps/chosen": -139.31689453125, + "logps/ref_chosen": -41.604888916015625, + "logps/ref_rejected": -77.51741027832031, + "logps/rejected": -275.88507080078125, + "loss": 1.0601, + "margin_dpo/margin_mean": 100.65567016601562, + "margin_dpo/margin_std": 133.34600830078125, + "step": 214 + }, + { + "KL/chosen_KL_mean": -108.55481719970703, + "KL/mean": -162.83993530273438, + "KL/rejected_KL_mean": -217.1250762939453, + "KL/std": 115.2216796875, + "epoch": 0.315712187958884, + "fcm_dpo/beta": 0.0045753284357488155, + "fcm_dpo/delta": -0.09945982694625854, + "fcm_dpo/margin": 108.57025909423828, + "fcm_dpo/q_t": 0.38728252053260803, + "grad_norm": 26.600690841674805, + "learning_rate": 4.3388522485142885e-07, + "logits/chosen": -0.30017614364624023, + "logits/rejected": -0.2858251929283142, + "logps/chosen": -161.83407592773438, + "logps/ref_chosen": -53.279266357421875, + "logps/ref_rejected": -89.96464538574219, + "logps/rejected": -307.0897216796875, + "loss": 1.0211, + "margin_dpo/margin_mean": 108.57025909423828, + "margin_dpo/margin_std": 121.32503509521484, + "step": 215 + }, + { + "KL/chosen_KL_mean": -110.91032409667969, + "KL/mean": -164.44400024414062, + "KL/rejected_KL_mean": -217.9776611328125, + "KL/std": 116.61207580566406, + "epoch": 0.31718061674008813, + "fcm_dpo/beta": 0.004546986427158117, + "fcm_dpo/delta": -0.08895479142665863, + "fcm_dpo/margin": 107.06733703613281, + "fcm_dpo/q_t": 0.391222208738327, + "grad_norm": 28.687511444091797, + "learning_rate": 4.330133748510036e-07, + "logits/chosen": -0.31943368911743164, + "logits/rejected": -0.30127909779548645, + "logps/chosen": -159.79812622070312, + "logps/ref_chosen": -48.887794494628906, + "logps/ref_rejected": -77.19892883300781, + "logps/rejected": -295.17657470703125, + "loss": 1.0535, + "margin_dpo/margin_mean": 107.06732940673828, + "margin_dpo/margin_std": 142.55877685546875, + "step": 216 + }, + { + "KL/chosen_KL_mean": -110.0457763671875, + "KL/mean": -171.7996826171875, + "KL/rejected_KL_mean": -233.55361938476562, + "KL/std": 119.768310546875, + "epoch": 0.3186490455212922, + "fcm_dpo/beta": 0.004474131390452385, + "fcm_dpo/delta": -0.15707343816757202, + "fcm_dpo/margin": 123.50784301757812, + "fcm_dpo/q_t": 0.376447856426239, + "grad_norm": 21.844968795776367, + "learning_rate": 4.3213670227794757e-07, + "logits/chosen": -0.32586967945098877, + "logits/rejected": -0.3191707134246826, + "logps/chosen": -159.89108276367188, + "logps/ref_chosen": -49.845306396484375, + "logps/ref_rejected": -100.07832336425781, + "logps/rejected": -333.6319580078125, + "loss": 0.9878, + "margin_dpo/margin_mean": 123.50784301757812, + "margin_dpo/margin_std": 134.11033630371094, + "step": 217 + }, + { + "KL/chosen_KL_mean": -115.66281127929688, + "KL/mean": -165.30850219726562, + "KL/rejected_KL_mean": -214.9542236328125, + "KL/std": 118.33514404296875, + "epoch": 0.3201174743024963, + "fcm_dpo/beta": 0.00444306805729866, + "fcm_dpo/delta": -0.04212786629796028, + "fcm_dpo/margin": 99.2914047241211, + "fcm_dpo/q_t": 0.40180084109306335, + "grad_norm": 22.195404052734375, + "learning_rate": 4.3125523023339815e-07, + "logits/chosen": -0.3458363711833954, + "logits/rejected": -0.33776670694351196, + "logps/chosen": -174.239501953125, + "logps/ref_chosen": -58.576683044433594, + "logps/ref_rejected": -87.84639739990234, + "logps/rejected": -302.80059814453125, + "loss": 1.0785, + "margin_dpo/margin_mean": 99.2914047241211, + "margin_dpo/margin_std": 137.7327423095703, + "step": 218 + }, + { + "KL/chosen_KL_mean": -124.46105194091797, + "KL/mean": -168.70489501953125, + "KL/rejected_KL_mean": -212.94874572753906, + "KL/std": 120.42536163330078, + "epoch": 0.32158590308370044, + "fcm_dpo/beta": 0.00445276964455843, + "fcm_dpo/delta": 0.005898263305425644, + "fcm_dpo/margin": 88.48770141601562, + "fcm_dpo/q_t": 0.4121992588043213, + "grad_norm": 22.444704055786133, + "learning_rate": 4.303689819449636e-07, + "logits/chosen": -0.3449735641479492, + "logits/rejected": -0.33431658148765564, + "logps/chosen": -185.54490661621094, + "logps/ref_chosen": -61.083858489990234, + "logps/ref_rejected": -85.83042907714844, + "logps/rejected": -298.7791748046875, + "loss": 1.1278, + "margin_dpo/margin_mean": 88.4876937866211, + "margin_dpo/margin_std": 145.41778564453125, + "step": 219 + }, + { + "KL/chosen_KL_mean": -146.02963256835938, + "KL/mean": -183.63616943359375, + "KL/rejected_KL_mean": -221.24269104003906, + "KL/std": 107.1483154296875, + "epoch": 0.32305433186490456, + "fcm_dpo/beta": 0.004458419047296047, + "fcm_dpo/delta": 0.06577261537313461, + "fcm_dpo/margin": 75.21304321289062, + "fcm_dpo/q_t": 0.42131632566452026, + "grad_norm": 22.67669677734375, + "learning_rate": 4.2947798076611047e-07, + "logits/chosen": -0.31147876381874084, + "logits/rejected": -0.2866259217262268, + "logps/chosen": -216.06092834472656, + "logps/ref_chosen": -70.03128051757812, + "logps/ref_rejected": -87.68551635742188, + "logps/rejected": -308.92822265625, + "loss": 1.1337, + "margin_dpo/margin_mean": 75.21304321289062, + "margin_dpo/margin_std": 106.4920654296875, + "step": 220 + }, + { + "KL/chosen_KL_mean": -136.16143798828125, + "KL/mean": -213.4010009765625, + "KL/rejected_KL_mean": -290.6405944824219, + "KL/std": 141.66920471191406, + "epoch": 0.3245227606461087, + "fcm_dpo/beta": 0.004385126754641533, + "fcm_dpo/delta": -0.2867964208126068, + "fcm_dpo/margin": 154.47915649414062, + "fcm_dpo/q_t": 0.3487062454223633, + "grad_norm": 29.397729873657227, + "learning_rate": 4.285822501755485e-07, + "logits/chosen": -0.2791799306869507, + "logits/rejected": -0.28570184111595154, + "logps/chosen": -188.31613159179688, + "logps/ref_chosen": -52.15470886230469, + "logps/ref_rejected": -106.46768188476562, + "logps/rejected": -397.1082763671875, + "loss": 0.9085, + "margin_dpo/margin_mean": 154.47915649414062, + "margin_dpo/margin_std": 143.16314697265625, + "step": 221 + }, + { + "KL/chosen_KL_mean": -145.255126953125, + "KL/mean": -204.62652587890625, + "KL/rejected_KL_mean": -263.99798583984375, + "KL/std": 128.56329345703125, + "epoch": 0.32599118942731276, + "fcm_dpo/beta": 0.004318445920944214, + "fcm_dpo/delta": -0.11569039523601532, + "fcm_dpo/margin": 118.74282836914062, + "fcm_dpo/q_t": 0.3848910629749298, + "grad_norm": 26.170949935913086, + "learning_rate": 4.276818137766118e-07, + "logits/chosen": -0.33129405975341797, + "logits/rejected": -0.32808414101600647, + "logps/chosen": -206.22622680664062, + "logps/ref_chosen": -60.971099853515625, + "logps/ref_rejected": -100.00115203857422, + "logps/rejected": -363.9991149902344, + "loss": 1.0234, + "margin_dpo/margin_mean": 118.74282836914062, + "margin_dpo/margin_std": 141.51174926757812, + "step": 222 + }, + { + "KL/chosen_KL_mean": -161.82325744628906, + "KL/mean": -213.307861328125, + "KL/rejected_KL_mean": -264.79248046875, + "KL/std": 131.9065399169922, + "epoch": 0.3274596182085169, + "fcm_dpo/beta": 0.004278381355106831, + "fcm_dpo/delta": -0.04155343025922775, + "fcm_dpo/margin": 102.96919250488281, + "fcm_dpo/q_t": 0.40245696902275085, + "grad_norm": 34.5849723815918, + "learning_rate": 4.2677669529663686e-07, + "logits/chosen": -0.2450229823589325, + "logits/rejected": -0.23633867502212524, + "logps/chosen": -214.46383666992188, + "logps/ref_chosen": -52.64057540893555, + "logps/ref_rejected": -82.82502746582031, + "logps/rejected": -347.61749267578125, + "loss": 1.1098, + "margin_dpo/margin_mean": 102.96919250488281, + "margin_dpo/margin_std": 167.57089233398438, + "step": 223 + }, + { + "KL/chosen_KL_mean": -148.254150390625, + "KL/mean": -204.7643280029297, + "KL/rejected_KL_mean": -261.2745361328125, + "KL/std": 144.42315673828125, + "epoch": 0.328928046989721, + "fcm_dpo/beta": 0.004234839230775833, + "fcm_dpo/delta": -0.08161487430334091, + "fcm_dpo/margin": 113.02037048339844, + "fcm_dpo/q_t": 0.39719724655151367, + "grad_norm": 37.836830139160156, + "learning_rate": 4.2586691858633747e-07, + "logits/chosen": -0.26897698640823364, + "logits/rejected": -0.24652792513370514, + "logps/chosen": -196.84954833984375, + "logps/ref_chosen": -48.59541320800781, + "logps/ref_rejected": -77.11648559570312, + "logps/rejected": -338.3910217285156, + "loss": 1.0801, + "margin_dpo/margin_mean": 113.02037048339844, + "margin_dpo/margin_std": 168.1210479736328, + "step": 224 + }, + { + "KL/chosen_KL_mean": -170.34971618652344, + "KL/mean": -240.19332885742188, + "KL/rejected_KL_mean": -310.03692626953125, + "KL/std": 145.15887451171875, + "epoch": 0.3303964757709251, + "fcm_dpo/beta": 0.0041815126314759254, + "fcm_dpo/delta": -0.1897869110107422, + "fcm_dpo/margin": 139.68719482421875, + "fcm_dpo/q_t": 0.37290918827056885, + "grad_norm": 30.018829345703125, + "learning_rate": 4.249525076191759e-07, + "logits/chosen": -0.2817176878452301, + "logits/rejected": -0.2691669464111328, + "logps/chosen": -228.35018920898438, + "logps/ref_chosen": -58.000465393066406, + "logps/ref_rejected": -99.90291595458984, + "logps/rejected": -409.9398193359375, + "loss": 1.0032, + "margin_dpo/margin_mean": 139.6872100830078, + "margin_dpo/margin_std": 175.74925231933594, + "step": 225 + }, + { + "KL/chosen_KL_mean": -143.76260375976562, + "KL/mean": -200.83154296875, + "KL/rejected_KL_mean": -257.90045166015625, + "KL/std": 143.9960479736328, + "epoch": 0.33186490455212925, + "fcm_dpo/beta": 0.004126027226448059, + "fcm_dpo/delta": -0.07347771525382996, + "fcm_dpo/margin": 114.13786315917969, + "fcm_dpo/q_t": 0.3982742428779602, + "grad_norm": 30.518173217773438, + "learning_rate": 4.2403348649073167e-07, + "logits/chosen": -0.35959500074386597, + "logits/rejected": -0.3191676139831543, + "logps/chosen": -202.66140747070312, + "logps/ref_chosen": -58.898799896240234, + "logps/ref_rejected": -78.68775939941406, + "logps/rejected": -336.5882263183594, + "loss": 1.0742, + "margin_dpo/margin_mean": 114.13786315917969, + "margin_dpo/margin_std": 164.3631591796875, + "step": 226 + }, + { + "KL/chosen_KL_mean": -155.82469177246094, + "KL/mean": -225.405029296875, + "KL/rejected_KL_mean": -294.9853515625, + "KL/std": 156.4764404296875, + "epoch": 0.3333333333333333, + "fcm_dpo/beta": 0.0040718745440244675, + "fcm_dpo/delta": -0.1720867156982422, + "fcm_dpo/margin": 139.16065979003906, + "fcm_dpo/q_t": 0.37408214807510376, + "grad_norm": 28.692975997924805, + "learning_rate": 4.2310987941806615e-07, + "logits/chosen": -0.3732532262802124, + "logits/rejected": -0.35874879360198975, + "logps/chosen": -214.89688110351562, + "logps/ref_chosen": -59.072181701660156, + "logps/ref_rejected": -99.41236877441406, + "logps/rejected": -394.3977355957031, + "loss": 0.9953, + "margin_dpo/margin_mean": 139.16065979003906, + "margin_dpo/margin_std": 163.25885009765625, + "step": 227 + }, + { + "KL/chosen_KL_mean": -154.69204711914062, + "KL/mean": -203.87847900390625, + "KL/rejected_KL_mean": -253.0648956298828, + "KL/std": 127.062255859375, + "epoch": 0.33480176211453744, + "fcm_dpo/beta": 0.004066440276801586, + "fcm_dpo/delta": -5.455687642097473e-05, + "fcm_dpo/margin": 98.37286376953125, + "fcm_dpo/q_t": 0.40937620401382446, + "grad_norm": 25.941659927368164, + "learning_rate": 4.2218171073908463e-07, + "logits/chosen": -0.3328084647655487, + "logits/rejected": -0.31316232681274414, + "logps/chosen": -220.5833282470703, + "logps/ref_chosen": -65.89128875732422, + "logps/ref_rejected": -91.04875183105469, + "logps/rejected": -344.1136474609375, + "loss": 1.1158, + "margin_dpo/margin_mean": 98.37286376953125, + "margin_dpo/margin_std": 152.87173461914062, + "step": 228 + }, + { + "KL/chosen_KL_mean": -146.78775024414062, + "KL/mean": -200.69815063476562, + "KL/rejected_KL_mean": -254.60853576660156, + "KL/std": 140.32000732421875, + "epoch": 0.33627019089574156, + "fcm_dpo/beta": 0.004051582887768745, + "fcm_dpo/delta": -0.03766883909702301, + "fcm_dpo/margin": 107.82078552246094, + "fcm_dpo/q_t": 0.4005894660949707, + "grad_norm": 39.35184860229492, + "learning_rate": 4.212490049118951e-07, + "logits/chosen": -0.409065306186676, + "logits/rejected": -0.3768808841705322, + "logps/chosen": -217.494140625, + "logps/ref_chosen": -70.70637512207031, + "logps/ref_rejected": -84.52741241455078, + "logps/rejected": -339.1359558105469, + "loss": 1.082, + "margin_dpo/margin_mean": 107.82078552246094, + "margin_dpo/margin_std": 151.07818603515625, + "step": 229 + }, + { + "KL/chosen_KL_mean": -113.96424102783203, + "KL/mean": -189.5445556640625, + "KL/rejected_KL_mean": -265.1248779296875, + "KL/std": 134.1195068359375, + "epoch": 0.3377386196769457, + "fcm_dpo/beta": 0.003982962109148502, + "fcm_dpo/delta": -0.2084825038909912, + "fcm_dpo/margin": 151.16061401367188, + "fcm_dpo/q_t": 0.3635350465774536, + "grad_norm": 26.65117645263672, + "learning_rate": 4.203117865141635e-07, + "logits/chosen": -0.28100255131721497, + "logits/rejected": -0.2840505540370941, + "logps/chosen": -153.24624633789062, + "logps/ref_chosen": -39.282005310058594, + "logps/ref_rejected": -85.62191009521484, + "logps/rejected": -350.74676513671875, + "loss": 0.9531, + "margin_dpo/margin_mean": 151.16061401367188, + "margin_dpo/margin_std": 151.04591369628906, + "step": 230 + }, + { + "KL/chosen_KL_mean": -125.48028564453125, + "KL/mean": -177.44976806640625, + "KL/rejected_KL_mean": -229.41925048828125, + "KL/std": 115.8082275390625, + "epoch": 0.3392070484581498, + "fcm_dpo/beta": 0.003960724454373121, + "fcm_dpo/delta": -0.011919192038476467, + "fcm_dpo/margin": 103.93896484375, + "fcm_dpo/q_t": 0.4071752727031708, + "grad_norm": 28.911128997802734, + "learning_rate": 4.1937008024246625e-07, + "logits/chosen": -0.3797636032104492, + "logits/rejected": -0.35482439398765564, + "logps/chosen": -188.75672912597656, + "logps/ref_chosen": -63.27644348144531, + "logps/ref_rejected": -74.1239013671875, + "logps/rejected": -303.54315185546875, + "loss": 1.0837, + "margin_dpo/margin_mean": 103.93896484375, + "margin_dpo/margin_std": 136.3351287841797, + "step": 231 + }, + { + "KL/chosen_KL_mean": -152.2584228515625, + "KL/mean": -195.20050048828125, + "KL/rejected_KL_mean": -238.14256286621094, + "KL/std": 134.2640838623047, + "epoch": 0.3406754772393539, + "fcm_dpo/beta": 0.0039717936888337135, + "fcm_dpo/delta": 0.059911780059337616, + "fcm_dpo/margin": 85.88412475585938, + "fcm_dpo/q_t": 0.4243953824043274, + "grad_norm": 27.307865142822266, + "learning_rate": 4.1842391091163933e-07, + "logits/chosen": -0.3296153247356415, + "logits/rejected": -0.3048354983329773, + "logps/chosen": -223.00718688964844, + "logps/ref_chosen": -70.74876403808594, + "logps/ref_rejected": -83.97706604003906, + "logps/rejected": -322.11962890625, + "loss": 1.1477, + "margin_dpo/margin_mean": 85.88412475585938, + "margin_dpo/margin_std": 142.83050537109375, + "step": 232 + }, + { + "KL/chosen_KL_mean": -135.91783142089844, + "KL/mean": -204.65484619140625, + "KL/rejected_KL_mean": -273.3918762207031, + "KL/std": 146.017333984375, + "epoch": 0.342143906020558, + "fcm_dpo/beta": 0.003943287767469883, + "fcm_dpo/delta": -0.14597077667713165, + "fcm_dpo/margin": 137.4740447998047, + "fcm_dpo/q_t": 0.38463860750198364, + "grad_norm": 24.649152755737305, + "learning_rate": 4.174733034541245e-07, + "logits/chosen": -0.34792935848236084, + "logits/rejected": -0.3480525016784668, + "logps/chosen": -190.80076599121094, + "logps/ref_chosen": -54.8829345703125, + "logps/ref_rejected": -107.4800796508789, + "logps/rejected": -380.8719482421875, + "loss": 1.0429, + "margin_dpo/margin_mean": 137.47402954101562, + "margin_dpo/margin_std": 192.8949737548828, + "step": 233 + }, + { + "KL/chosen_KL_mean": -127.90108489990234, + "KL/mean": -198.03463745117188, + "KL/rejected_KL_mean": -268.168212890625, + "KL/std": 129.72488403320312, + "epoch": 0.3436123348017621, + "fcm_dpo/beta": 0.0038681491278111935, + "fcm_dpo/delta": -0.14766976237297058, + "fcm_dpo/margin": 140.26710510253906, + "fcm_dpo/q_t": 0.3764752745628357, + "grad_norm": 24.525766372680664, + "learning_rate": 4.165182829193126e-07, + "logits/chosen": -0.3588540554046631, + "logits/rejected": -0.3863643705844879, + "logps/chosen": -171.99560546875, + "logps/ref_chosen": -44.094520568847656, + "logps/ref_rejected": -100.00663757324219, + "logps/rejected": -368.1748352050781, + "loss": 0.9883, + "margin_dpo/margin_mean": 140.26710510253906, + "margin_dpo/margin_std": 142.52896118164062, + "step": 234 + }, + { + "KL/chosen_KL_mean": -163.20513916015625, + "KL/mean": -208.35916137695312, + "KL/rejected_KL_mean": -253.51319885253906, + "KL/std": 124.54771423339844, + "epoch": 0.34508076358296624, + "fcm_dpo/beta": 0.0038875495083630085, + "fcm_dpo/delta": 0.049665797501802444, + "fcm_dpo/margin": 90.30805969238281, + "fcm_dpo/q_t": 0.42041072249412537, + "grad_norm": 26.740400314331055, + "learning_rate": 4.1555887447288255e-07, + "logits/chosen": -0.3990609049797058, + "logits/rejected": -0.37867674231529236, + "logps/chosen": -225.44305419921875, + "logps/ref_chosen": -62.237911224365234, + "logps/ref_rejected": -90.39506530761719, + "logps/rejected": -343.90826416015625, + "loss": 1.1485, + "margin_dpo/margin_mean": 90.30805969238281, + "margin_dpo/margin_std": 152.52548217773438, + "step": 235 + }, + { + "KL/chosen_KL_mean": -124.98179626464844, + "KL/mean": -199.99957275390625, + "KL/rejected_KL_mean": -275.0173645019531, + "KL/std": 138.9434051513672, + "epoch": 0.3465491923641703, + "fcm_dpo/beta": 0.0038370825350284576, + "fcm_dpo/delta": -0.18083232641220093, + "fcm_dpo/margin": 150.0355682373047, + "fcm_dpo/q_t": 0.3679265081882477, + "grad_norm": 33.604164123535156, + "learning_rate": 4.1459510339613946e-07, + "logits/chosen": -0.37230125069618225, + "logits/rejected": -0.3716471195220947, + "logps/chosen": -174.32315063476562, + "logps/ref_chosen": -49.34136199951172, + "logps/ref_rejected": -103.51162719726562, + "logps/rejected": -378.52899169921875, + "loss": 0.9526, + "margin_dpo/margin_mean": 150.0355682373047, + "margin_dpo/margin_std": 135.6669464111328, + "step": 236 + }, + { + "KL/chosen_KL_mean": -173.83096313476562, + "KL/mean": -229.819580078125, + "KL/rejected_KL_mean": -285.8081970214844, + "KL/std": 140.05715942382812, + "epoch": 0.34801762114537443, + "fcm_dpo/beta": 0.003812191542237997, + "fcm_dpo/delta": -0.027462899684906006, + "fcm_dpo/margin": 111.97723388671875, + "fcm_dpo/q_t": 0.40431371331214905, + "grad_norm": 30.839101791381836, + "learning_rate": 4.136269950853473e-07, + "logits/chosen": -0.3602313995361328, + "logits/rejected": -0.35376355051994324, + "logps/chosen": -227.99908447265625, + "logps/ref_chosen": -54.168121337890625, + "logps/ref_rejected": -94.78036499023438, + "logps/rejected": -380.58856201171875, + "loss": 1.0919, + "margin_dpo/margin_mean": 111.97724151611328, + "margin_dpo/margin_std": 162.9444580078125, + "step": 237 + }, + { + "KL/chosen_KL_mean": -161.74685668945312, + "KL/mean": -218.63809204101562, + "KL/rejected_KL_mean": -275.5293273925781, + "KL/std": 138.28067016601562, + "epoch": 0.34948604992657856, + "fcm_dpo/beta": 0.0037880742456763983, + "fcm_dpo/delta": -0.03207089379429817, + "fcm_dpo/margin": 113.78250122070312, + "fcm_dpo/q_t": 0.4048614501953125, + "grad_norm": 25.45376968383789, + "learning_rate": 4.126545750510605e-07, + "logits/chosen": -0.3611815571784973, + "logits/rejected": -0.3783150911331177, + "logps/chosen": -215.719970703125, + "logps/ref_chosen": -53.973121643066406, + "logps/ref_rejected": -89.41795349121094, + "logps/rejected": -364.947265625, + "loss": 1.0809, + "margin_dpo/margin_mean": 113.7824935913086, + "margin_dpo/margin_std": 156.38768005371094, + "step": 238 + }, + { + "KL/chosen_KL_mean": -173.99893188476562, + "KL/mean": -237.2444305419922, + "KL/rejected_KL_mean": -300.4898986816406, + "KL/std": 130.01622009277344, + "epoch": 0.3509544787077827, + "fcm_dpo/beta": 0.0037591117434203625, + "fcm_dpo/delta": -0.07816985249519348, + "fcm_dpo/margin": 126.49100494384766, + "fcm_dpo/q_t": 0.39054858684539795, + "grad_norm": 27.46272087097168, + "learning_rate": 4.116778689174514e-07, + "logits/chosen": -0.3645399808883667, + "logits/rejected": -0.34787994623184204, + "logps/chosen": -232.09674072265625, + "logps/ref_chosen": -58.09782409667969, + "logps/ref_rejected": -93.59294128417969, + "logps/rejected": -394.0828552246094, + "loss": 1.0356, + "margin_dpo/margin_mean": 126.49099731445312, + "margin_dpo/margin_std": 139.7375030517578, + "step": 239 + }, + { + "KL/chosen_KL_mean": -181.10501098632812, + "KL/mean": -235.21560668945312, + "KL/rejected_KL_mean": -289.326171875, + "KL/std": 143.65615844726562, + "epoch": 0.3524229074889868, + "fcm_dpo/beta": 0.003757942235097289, + "fcm_dpo/delta": -0.006928959861397743, + "fcm_dpo/margin": 108.22117614746094, + "fcm_dpo/q_t": 0.40816569328308105, + "grad_norm": 32.250770568847656, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": -0.35770127177238464, + "logits/rejected": -0.3304799795150757, + "logps/chosen": -241.719482421875, + "logps/ref_chosen": -60.6144905090332, + "logps/ref_rejected": -74.1185302734375, + "logps/rejected": -363.4447021484375, + "loss": 1.1151, + "margin_dpo/margin_mean": 108.2211685180664, + "margin_dpo/margin_std": 168.62789916992188, + "step": 240 + }, + { + "KL/chosen_KL_mean": -149.57997131347656, + "KL/mean": -231.7826690673828, + "KL/rejected_KL_mean": -313.9853820800781, + "KL/std": 156.34739685058594, + "epoch": 0.35389133627019087, + "fcm_dpo/beta": 0.0037074610590934753, + "fcm_dpo/delta": -0.21590769290924072, + "fcm_dpo/margin": 164.4053955078125, + "fcm_dpo/q_t": 0.3671102821826935, + "grad_norm": 24.17862892150879, + "learning_rate": 4.097117014129903e-07, + "logits/chosen": -0.4300222396850586, + "logits/rejected": -0.4012630581855774, + "logps/chosen": -215.67105102539062, + "logps/ref_chosen": -66.091064453125, + "logps/ref_rejected": -88.06088256835938, + "logps/rejected": -402.0462646484375, + "loss": 0.9689, + "margin_dpo/margin_mean": 164.4053955078125, + "margin_dpo/margin_std": 181.3541717529297, + "step": 241 + }, + { + "KL/chosen_KL_mean": -168.50222778320312, + "KL/mean": -227.4334716796875, + "KL/rejected_KL_mean": -286.3647155761719, + "KL/std": 131.76930236816406, + "epoch": 0.355359765051395, + "fcm_dpo/beta": 0.0036807293072342873, + "fcm_dpo/delta": -0.03467430919408798, + "fcm_dpo/margin": 117.86245727539062, + "fcm_dpo/q_t": 0.4024999737739563, + "grad_norm": 26.07633399963379, + "learning_rate": 4.087222918524807e-07, + "logits/chosen": -0.35617491602897644, + "logits/rejected": -0.33020225167274475, + "logps/chosen": -236.3661651611328, + "logps/ref_chosen": -67.86392974853516, + "logps/ref_rejected": -83.36033630371094, + "logps/rejected": -369.72503662109375, + "loss": 1.0726, + "margin_dpo/margin_mean": 117.86245727539062, + "margin_dpo/margin_std": 153.87033081054688, + "step": 242 + }, + { + "KL/chosen_KL_mean": -169.4001922607422, + "KL/mean": -241.8457794189453, + "KL/rejected_KL_mean": -314.2913818359375, + "KL/std": 151.13641357421875, + "epoch": 0.3568281938325991, + "fcm_dpo/beta": 0.003632163628935814, + "fcm_dpo/delta": -0.12977714836597443, + "fcm_dpo/margin": 144.89117431640625, + "fcm_dpo/q_t": 0.3812503218650818, + "grad_norm": 23.20541763305664, + "learning_rate": 4.07728699811968e-07, + "logits/chosen": -0.3599437475204468, + "logits/rejected": -0.32367074489593506, + "logps/chosen": -232.48443603515625, + "logps/ref_chosen": -63.0842399597168, + "logps/ref_rejected": -76.33563232421875, + "logps/rejected": -390.62701416015625, + "loss": 1.0077, + "margin_dpo/margin_mean": 144.89117431640625, + "margin_dpo/margin_std": 165.2962646484375, + "step": 243 + }, + { + "KL/chosen_KL_mean": -147.78445434570312, + "KL/mean": -222.11541748046875, + "KL/rejected_KL_mean": -296.4464111328125, + "KL/std": 144.9271240234375, + "epoch": 0.35829662261380324, + "fcm_dpo/beta": 0.0035813949070870876, + "fcm_dpo/delta": -0.13626864552497864, + "fcm_dpo/margin": 148.6619415283203, + "fcm_dpo/q_t": 0.3784053921699524, + "grad_norm": 27.900978088378906, + "learning_rate": 4.067309514735267e-07, + "logits/chosen": -0.42759430408477783, + "logits/rejected": -0.41793277859687805, + "logps/chosen": -208.92514038085938, + "logps/ref_chosen": -61.140689849853516, + "logps/ref_rejected": -94.89193725585938, + "logps/rejected": -391.33831787109375, + "loss": 0.9878, + "margin_dpo/margin_mean": 148.66195678710938, + "margin_dpo/margin_std": 146.8434295654297, + "step": 244 + }, + { + "KL/chosen_KL_mean": -168.794677734375, + "KL/mean": -229.46194458007812, + "KL/rejected_KL_mean": -290.12921142578125, + "KL/std": 146.95806884765625, + "epoch": 0.35976505139500736, + "fcm_dpo/beta": 0.003548422595486045, + "fcm_dpo/delta": -0.03223326802253723, + "fcm_dpo/margin": 121.33451843261719, + "fcm_dpo/q_t": 0.4033067524433136, + "grad_norm": 29.785207748413086, + "learning_rate": 4.057290731287531e-07, + "logits/chosen": -0.4032437801361084, + "logits/rejected": -0.3697701692581177, + "logps/chosen": -236.05697631835938, + "logps/ref_chosen": -67.26228332519531, + "logps/ref_rejected": -87.64010620117188, + "logps/rejected": -377.769287109375, + "loss": 1.0876, + "margin_dpo/margin_mean": 121.33451843261719, + "margin_dpo/margin_std": 165.6243438720703, + "step": 245 + }, + { + "KL/chosen_KL_mean": -169.9764404296875, + "KL/mean": -230.96017456054688, + "KL/rejected_KL_mean": -291.94390869140625, + "KL/std": 162.06460571289062, + "epoch": 0.36123348017621143, + "fcm_dpo/beta": 0.003547412808984518, + "fcm_dpo/delta": -0.0335458442568779, + "fcm_dpo/margin": 121.96745300292969, + "fcm_dpo/q_t": 0.4048032760620117, + "grad_norm": 29.99637222290039, + "learning_rate": 4.047230911780736e-07, + "logits/chosen": -0.4445304274559021, + "logits/rejected": -0.4077053964138031, + "logps/chosen": -236.6734161376953, + "logps/ref_chosen": -66.69696807861328, + "logps/ref_rejected": -84.34634399414062, + "logps/rejected": -376.2902526855469, + "loss": 1.0863, + "margin_dpo/margin_mean": 121.96745300292969, + "margin_dpo/margin_std": 176.64109802246094, + "step": 246 + }, + { + "KL/chosen_KL_mean": -202.48690795898438, + "KL/mean": -290.2227478027344, + "KL/rejected_KL_mean": -377.9586181640625, + "KL/std": 172.92800903320312, + "epoch": 0.36270190895741555, + "fcm_dpo/beta": 0.0034898994490504265, + "fcm_dpo/delta": -0.21926915645599365, + "fcm_dpo/margin": 175.4716796875, + "fcm_dpo/q_t": 0.3639224171638489, + "grad_norm": 31.32638931274414, + "learning_rate": 4.0371303213004814e-07, + "logits/chosen": -0.34930846095085144, + "logits/rejected": -0.3457414507865906, + "logps/chosen": -259.0922546386719, + "logps/ref_chosen": -56.6053466796875, + "logps/ref_rejected": -106.29326629638672, + "logps/rejected": -484.2518615722656, + "loss": 0.9621, + "margin_dpo/margin_mean": 175.4716796875, + "margin_dpo/margin_std": 188.98797607421875, + "step": 247 + }, + { + "KL/chosen_KL_mean": -176.61392211914062, + "KL/mean": -252.390625, + "KL/rejected_KL_mean": -328.1673278808594, + "KL/std": 141.5336151123047, + "epoch": 0.3641703377386197, + "fcm_dpo/beta": 0.003434403333812952, + "fcm_dpo/delta": -0.12410237640142441, + "fcm_dpo/margin": 151.55343627929688, + "fcm_dpo/q_t": 0.3788222670555115, + "grad_norm": 26.931861877441406, + "learning_rate": 4.0269892260067197e-07, + "logits/chosen": -0.35363906621932983, + "logits/rejected": -0.37153977155685425, + "logps/chosen": -220.65713500976562, + "logps/ref_chosen": -44.043216705322266, + "logps/ref_rejected": -91.85687255859375, + "logps/rejected": -420.02423095703125, + "loss": 0.9855, + "margin_dpo/margin_mean": 151.55343627929688, + "margin_dpo/margin_std": 138.50741577148438, + "step": 248 + }, + { + "KL/chosen_KL_mean": -223.4722137451172, + "KL/mean": -264.677734375, + "KL/rejected_KL_mean": -305.8832702636719, + "KL/std": 150.7882080078125, + "epoch": 0.3656387665198238, + "fcm_dpo/beta": 0.0034578610211610794, + "fcm_dpo/delta": 0.11675933003425598, + "fcm_dpo/margin": 82.41104888916016, + "fcm_dpo/q_t": 0.43573471903800964, + "grad_norm": 46.4980583190918, + "learning_rate": 4.0168078931267426e-07, + "logits/chosen": -0.3905929923057556, + "logits/rejected": -0.3669302463531494, + "logps/chosen": -285.91455078125, + "logps/ref_chosen": -62.442352294921875, + "logps/ref_rejected": -80.46806335449219, + "logps/rejected": -386.351318359375, + "loss": 1.2111, + "margin_dpo/margin_mean": 82.41105651855469, + "margin_dpo/margin_std": 178.0140380859375, + "step": 249 + }, + { + "KL/chosen_KL_mean": -191.8778076171875, + "KL/mean": -268.19073486328125, + "KL/rejected_KL_mean": -344.503662109375, + "KL/std": 147.71640014648438, + "epoch": 0.3671071953010279, + "fcm_dpo/beta": 0.003444279544055462, + "fcm_dpo/delta": -0.12898439168930054, + "fcm_dpo/margin": 152.62586975097656, + "fcm_dpo/q_t": 0.3783637285232544, + "grad_norm": 38.140594482421875, + "learning_rate": 4.006586590948141e-07, + "logits/chosen": -0.37574708461761475, + "logits/rejected": -0.3200438618659973, + "logps/chosen": -257.5145263671875, + "logps/ref_chosen": -65.63668823242188, + "logps/ref_rejected": -73.87184143066406, + "logps/rejected": -418.3755187988281, + "loss": 0.9902, + "margin_dpo/margin_mean": 152.62586975097656, + "margin_dpo/margin_std": 148.07838439941406, + "step": 250 + }, + { + "KL/chosen_KL_mean": -210.373291015625, + "KL/mean": -262.0963134765625, + "KL/rejected_KL_mean": -313.8193664550781, + "KL/std": 158.57400512695312, + "epoch": 0.368575624082232, + "fcm_dpo/beta": 0.003431498073041439, + "fcm_dpo/delta": 0.045787617564201355, + "fcm_dpo/margin": 103.44607543945312, + "fcm_dpo/q_t": 0.4200136065483093, + "grad_norm": 31.18090057373047, + "learning_rate": 3.9963255888117325e-07, + "logits/chosen": -0.3738080859184265, + "logits/rejected": -0.34553277492523193, + "logps/chosen": -267.5559997558594, + "logps/ref_chosen": -57.182716369628906, + "logps/ref_rejected": -77.66343688964844, + "logps/rejected": -391.4827880859375, + "loss": 1.1472, + "margin_dpo/margin_mean": 103.44607543945312, + "margin_dpo/margin_std": 171.2592010498047, + "step": 251 + }, + { + "KL/chosen_KL_mean": -202.80035400390625, + "KL/mean": -273.161865234375, + "KL/rejected_KL_mean": -343.52337646484375, + "KL/std": 140.35821533203125, + "epoch": 0.3700440528634361, + "fcm_dpo/beta": 0.003416198305785656, + "fcm_dpo/delta": -0.08289383351802826, + "fcm_dpo/margin": 140.72303771972656, + "fcm_dpo/q_t": 0.38559865951538086, + "grad_norm": 29.2058048248291, + "learning_rate": 3.9860251571044666e-07, + "logits/chosen": -0.413099467754364, + "logits/rejected": -0.3722429573535919, + "logps/chosen": -274.4859924316406, + "logps/ref_chosen": -71.68563842773438, + "logps/ref_rejected": -84.75799560546875, + "logps/rejected": -428.2813720703125, + "loss": 1.0164, + "margin_dpo/margin_mean": 140.72303771972656, + "margin_dpo/margin_std": 137.57774353027344, + "step": 252 + }, + { + "KL/chosen_KL_mean": -172.536865234375, + "KL/mean": -240.89328002929688, + "KL/rejected_KL_mean": -309.2496643066406, + "KL/std": 148.74118041992188, + "epoch": 0.37151248164464024, + "fcm_dpo/beta": 0.0034063609782606363, + "fcm_dpo/delta": -0.06741885095834732, + "fcm_dpo/margin": 136.71279907226562, + "fcm_dpo/q_t": 0.392569899559021, + "grad_norm": 23.644466400146484, + "learning_rate": 3.9756855672522986e-07, + "logits/chosen": -0.414895236492157, + "logits/rejected": -0.4082959294319153, + "logps/chosen": -241.67080688476562, + "logps/ref_chosen": -69.1339340209961, + "logps/ref_rejected": -98.70252990722656, + "logps/rejected": -407.95220947265625, + "loss": 1.0485, + "margin_dpo/margin_mean": 136.71279907226562, + "margin_dpo/margin_std": 162.5023193359375, + "step": 253 + }, + { + "KL/chosen_KL_mean": -175.3002166748047, + "KL/mean": -234.9326171875, + "KL/rejected_KL_mean": -294.5650329589844, + "KL/std": 157.5005645751953, + "epoch": 0.37298091042584436, + "fcm_dpo/beta": 0.003387467935681343, + "fcm_dpo/delta": -0.0041029490530490875, + "fcm_dpo/margin": 119.26481628417969, + "fcm_dpo/q_t": 0.4124619662761688, + "grad_norm": 26.61585235595703, + "learning_rate": 3.965307091713037e-07, + "logits/chosen": -0.377765953540802, + "logits/rejected": -0.3579869270324707, + "logps/chosen": -229.45521545410156, + "logps/ref_chosen": -54.154998779296875, + "logps/ref_rejected": -90.30764770507812, + "logps/rejected": -384.8726806640625, + "loss": 1.1203, + "margin_dpo/margin_mean": 119.26481628417969, + "margin_dpo/margin_std": 196.88693237304688, + "step": 254 + }, + { + "KL/chosen_KL_mean": -181.1912841796875, + "KL/mean": -242.2811279296875, + "KL/rejected_KL_mean": -303.3709716796875, + "KL/std": 136.550048828125, + "epoch": 0.3744493392070485, + "fcm_dpo/beta": 0.003378466470167041, + "fcm_dpo/delta": -0.013259213417768478, + "fcm_dpo/margin": 122.17970275878906, + "fcm_dpo/q_t": 0.4052705764770508, + "grad_norm": 23.39793586730957, + "learning_rate": 3.954890003969163e-07, + "logits/chosen": -0.36594948172569275, + "logits/rejected": -0.35779714584350586, + "logps/chosen": -238.33294677734375, + "logps/ref_chosen": -57.14167022705078, + "logps/ref_rejected": -90.2085952758789, + "logps/rejected": -393.57958984375, + "loss": 1.0967, + "margin_dpo/margin_mean": 122.17970275878906, + "margin_dpo/margin_std": 172.21868896484375, + "step": 255 + }, + { + "KL/chosen_KL_mean": -156.4071044921875, + "KL/mean": -223.8316650390625, + "KL/rejected_KL_mean": -291.2562561035156, + "KL/std": 148.6212158203125, + "epoch": 0.37591776798825255, + "fcm_dpo/beta": 0.003370137419551611, + "fcm_dpo/delta": -0.05573172867298126, + "fcm_dpo/margin": 134.84915161132812, + "fcm_dpo/q_t": 0.3962138891220093, + "grad_norm": 22.66250228881836, + "learning_rate": 3.944434578520628e-07, + "logits/chosen": -0.3113047480583191, + "logits/rejected": -0.31630462408065796, + "logps/chosen": -211.57058715820312, + "logps/ref_chosen": -55.163490295410156, + "logps/ref_rejected": -92.56291961669922, + "logps/rejected": -383.81915283203125, + "loss": 1.0567, + "margin_dpo/margin_mean": 134.84913635253906, + "margin_dpo/margin_std": 168.68287658691406, + "step": 256 + }, + { + "KL/chosen_KL_mean": -151.5054931640625, + "KL/mean": -221.162109375, + "KL/rejected_KL_mean": -290.8187255859375, + "KL/std": 151.1150665283203, + "epoch": 0.37738619676945667, + "fcm_dpo/beta": 0.0033370412420481443, + "fcm_dpo/delta": -0.06718017160892487, + "fcm_dpo/margin": 139.313232421875, + "fcm_dpo/q_t": 0.39490461349487305, + "grad_norm": 20.624574661254883, + "learning_rate": 3.933941090877615e-07, + "logits/chosen": -0.32282236218452454, + "logits/rejected": -0.3079400360584259, + "logps/chosen": -200.92919921875, + "logps/ref_chosen": -49.42369842529297, + "logps/ref_rejected": -79.53791809082031, + "logps/rejected": -370.35662841796875, + "loss": 1.0528, + "margin_dpo/margin_mean": 139.313232421875, + "margin_dpo/margin_std": 168.8369598388672, + "step": 257 + }, + { + "KL/chosen_KL_mean": -196.22232055664062, + "KL/mean": -263.02752685546875, + "KL/rejected_KL_mean": -329.832763671875, + "KL/std": 156.14871215820312, + "epoch": 0.3788546255506608, + "fcm_dpo/beta": 0.0033341706730425358, + "fcm_dpo/delta": -0.0465044304728508, + "fcm_dpo/margin": 133.61041259765625, + "fcm_dpo/q_t": 0.3972247242927551, + "grad_norm": 24.023584365844727, + "learning_rate": 3.923409817553284e-07, + "logits/chosen": -0.316849946975708, + "logits/rejected": -0.3123030960559845, + "logps/chosen": -255.6064453125, + "logps/ref_chosen": -59.384124755859375, + "logps/ref_rejected": -95.99010467529297, + "logps/rejected": -425.8228454589844, + "loss": 1.0785, + "margin_dpo/margin_mean": 133.61041259765625, + "margin_dpo/margin_std": 185.94827270507812, + "step": 258 + }, + { + "KL/chosen_KL_mean": -187.66934204101562, + "KL/mean": -247.58529663085938, + "KL/rejected_KL_mean": -307.50128173828125, + "KL/std": 151.50286865234375, + "epoch": 0.3803230543318649, + "fcm_dpo/beta": 0.0033223240170627832, + "fcm_dpo/delta": 0.001799125224351883, + "fcm_dpo/margin": 119.83191680908203, + "fcm_dpo/q_t": 0.409615159034729, + "grad_norm": 26.094667434692383, + "learning_rate": 3.9128410360564793e-07, + "logits/chosen": -0.3730711042881012, + "logits/rejected": -0.36661672592163086, + "logps/chosen": -240.4976806640625, + "logps/ref_chosen": -52.828346252441406, + "logps/ref_rejected": -89.191650390625, + "logps/rejected": -396.69293212890625, + "loss": 1.1018, + "margin_dpo/margin_mean": 119.83192443847656, + "margin_dpo/margin_std": 169.11459350585938, + "step": 259 + }, + { + "KL/chosen_KL_mean": -195.3187255859375, + "KL/mean": -271.7413330078125, + "KL/rejected_KL_mean": -348.1639404296875, + "KL/std": 158.65463256835938, + "epoch": 0.38179148311306904, + "fcm_dpo/beta": 0.003307567210868001, + "fcm_dpo/delta": -0.10822096467018127, + "fcm_dpo/margin": 152.84521484375, + "fcm_dpo/q_t": 0.3863574266433716, + "grad_norm": 33.96175003051758, + "learning_rate": 3.9022350248844246e-07, + "logits/chosen": -0.35657697916030884, + "logits/rejected": -0.3744693994522095, + "logps/chosen": -242.73638916015625, + "logps/ref_chosen": -47.41767501831055, + "logps/ref_rejected": -95.08978271484375, + "logps/rejected": -443.25372314453125, + "loss": 1.013, + "margin_dpo/margin_mean": 152.84521484375, + "margin_dpo/margin_std": 169.8836212158203, + "step": 260 + }, + { + "KL/chosen_KL_mean": -197.50112915039062, + "KL/mean": -269.7708435058594, + "KL/rejected_KL_mean": -342.04052734375, + "KL/std": 168.48382568359375, + "epoch": 0.3832599118942731, + "fcm_dpo/beta": 0.0032701774034649134, + "fcm_dpo/delta": -0.07457753270864487, + "fcm_dpo/margin": 144.5394287109375, + "fcm_dpo/q_t": 0.39346715807914734, + "grad_norm": 23.455717086791992, + "learning_rate": 3.891592063515376e-07, + "logits/chosen": -0.3204799294471741, + "logits/rejected": -0.31587928533554077, + "logps/chosen": -250.5325164794922, + "logps/ref_chosen": -53.03137969970703, + "logps/ref_rejected": -88.51494598388672, + "logps/rejected": -430.55548095703125, + "loss": 1.0506, + "margin_dpo/margin_mean": 144.5394287109375, + "margin_dpo/margin_std": 183.651611328125, + "step": 261 + }, + { + "KL/chosen_KL_mean": -230.0545654296875, + "KL/mean": -290.9325256347656, + "KL/rejected_KL_mean": -351.8105163574219, + "KL/std": 155.808349609375, + "epoch": 0.38472834067547723, + "fcm_dpo/beta": 0.0032733541447669268, + "fcm_dpo/delta": 0.0013803504407405853, + "fcm_dpo/margin": 121.75595092773438, + "fcm_dpo/q_t": 0.4075956344604492, + "grad_norm": 29.473356246948242, + "learning_rate": 3.880912432401264e-07, + "logits/chosen": -0.3197232484817505, + "logits/rejected": -0.28875744342803955, + "logps/chosen": -289.6746826171875, + "logps/ref_chosen": -59.620140075683594, + "logps/ref_rejected": -86.41853332519531, + "logps/rejected": -438.22906494140625, + "loss": 1.086, + "margin_dpo/margin_mean": 121.75595092773438, + "margin_dpo/margin_std": 153.61898803710938, + "step": 262 + }, + { + "KL/chosen_KL_mean": -204.87518310546875, + "KL/mean": -291.5097351074219, + "KL/rejected_KL_mean": -378.1442565917969, + "KL/std": 180.58358764648438, + "epoch": 0.38619676945668135, + "fcm_dpo/beta": 0.003221530467271805, + "fcm_dpo/delta": -0.16330862045288086, + "fcm_dpo/margin": 173.26904296875, + "fcm_dpo/q_t": 0.3738434314727783, + "grad_norm": 23.88220977783203, + "learning_rate": 3.870196412960302e-07, + "logits/chosen": -0.3573540151119232, + "logits/rejected": -0.3329017460346222, + "logps/chosen": -264.296142578125, + "logps/ref_chosen": -59.42094421386719, + "logps/ref_rejected": -96.85720825195312, + "logps/rejected": -475.00146484375, + "loss": 0.9867, + "margin_dpo/margin_mean": 173.26905822753906, + "margin_dpo/margin_std": 186.5802764892578, + "step": 263 + }, + { + "KL/chosen_KL_mean": -210.99795532226562, + "KL/mean": -283.88690185546875, + "KL/rejected_KL_mean": -356.77581787109375, + "KL/std": 164.59521484375, + "epoch": 0.3876651982378855, + "fcm_dpo/beta": 0.0031880121678113937, + "fcm_dpo/delta": -0.06703611463308334, + "fcm_dpo/margin": 145.77786254882812, + "fcm_dpo/q_t": 0.39508941769599915, + "grad_norm": 27.110586166381836, + "learning_rate": 3.8594442875695665e-07, + "logits/chosen": -0.374176949262619, + "logits/rejected": -0.3629017472267151, + "logps/chosen": -273.72003173828125, + "logps/ref_chosen": -62.722084045410156, + "logps/ref_rejected": -93.85620880126953, + "logps/rejected": -450.6320495605469, + "loss": 1.0561, + "margin_dpo/margin_mean": 145.77786254882812, + "margin_dpo/margin_std": 182.66200256347656, + "step": 264 + }, + { + "KL/chosen_KL_mean": -237.04635620117188, + "KL/mean": -308.437744140625, + "KL/rejected_KL_mean": -379.8291015625, + "KL/std": 194.23110961914062, + "epoch": 0.3891336270190896, + "fcm_dpo/beta": 0.0031799268908798695, + "fcm_dpo/delta": -0.0553123913705349, + "fcm_dpo/margin": 142.78271484375, + "fcm_dpo/q_t": 0.40076446533203125, + "grad_norm": 29.41632843017578, + "learning_rate": 3.848656339557562e-07, + "logits/chosen": -0.33580052852630615, + "logits/rejected": -0.31790751218795776, + "logps/chosen": -299.017822265625, + "logps/ref_chosen": -61.971466064453125, + "logps/ref_rejected": -88.02059936523438, + "logps/rejected": -467.8497009277344, + "loss": 1.0958, + "margin_dpo/margin_mean": 142.78274536132812, + "margin_dpo/margin_std": 224.72232055664062, + "step": 265 + }, + { + "KL/chosen_KL_mean": -247.69691467285156, + "KL/mean": -305.4302978515625, + "KL/rejected_KL_mean": -363.1636962890625, + "KL/std": 160.6911163330078, + "epoch": 0.39060205580029367, + "fcm_dpo/beta": 0.0031850021332502365, + "fcm_dpo/delta": 0.03283064067363739, + "fcm_dpo/margin": 115.46676635742188, + "fcm_dpo/q_t": 0.4174157381057739, + "grad_norm": 63.25857925415039, + "learning_rate": 3.8378328531967507e-07, + "logits/chosen": -0.36059173941612244, + "logits/rejected": -0.3145703673362732, + "logps/chosen": -314.79656982421875, + "logps/ref_chosen": -67.09967041015625, + "logps/ref_rejected": -67.97122192382812, + "logps/rejected": -431.1349182128906, + "loss": 1.1341, + "margin_dpo/margin_mean": 115.46676635742188, + "margin_dpo/margin_std": 187.57647705078125, + "step": 266 + }, + { + "KL/chosen_KL_mean": -224.26531982421875, + "KL/mean": -294.53692626953125, + "KL/rejected_KL_mean": -364.8084716796875, + "KL/std": 172.51197814941406, + "epoch": 0.3920704845814978, + "fcm_dpo/beta": 0.00316983787342906, + "fcm_dpo/delta": -0.046759672462940216, + "fcm_dpo/margin": 140.54315185546875, + "fcm_dpo/q_t": 0.39952901005744934, + "grad_norm": 43.019325256347656, + "learning_rate": 3.8269741136960646e-07, + "logits/chosen": -0.3794315457344055, + "logits/rejected": -0.342626690864563, + "logps/chosen": -293.236083984375, + "logps/ref_chosen": -68.97075653076172, + "logps/ref_rejected": -90.16844940185547, + "logps/rejected": -454.9769287109375, + "loss": 1.0756, + "margin_dpo/margin_mean": 140.5431365966797, + "margin_dpo/margin_std": 194.42465209960938, + "step": 267 + }, + { + "KL/chosen_KL_mean": -234.8605194091797, + "KL/mean": -304.49365234375, + "KL/rejected_KL_mean": -374.12677001953125, + "KL/std": 169.82725524902344, + "epoch": 0.3935389133627019, + "fcm_dpo/beta": 0.0031569194979965687, + "fcm_dpo/delta": -0.04073052108287811, + "fcm_dpo/margin": 139.26629638671875, + "fcm_dpo/q_t": 0.402080237865448, + "grad_norm": 35.164588928222656, + "learning_rate": 3.8160804071933894e-07, + "logits/chosen": -0.36576220393180847, + "logits/rejected": -0.3701912760734558, + "logps/chosen": -290.7608337402344, + "logps/ref_chosen": -55.90031051635742, + "logps/ref_rejected": -101.64763641357422, + "logps/rejected": -475.7744140625, + "loss": 1.0845, + "margin_dpo/margin_mean": 139.26629638671875, + "margin_dpo/margin_std": 201.86578369140625, + "step": 268 + }, + { + "KL/chosen_KL_mean": -243.96658325195312, + "KL/mean": -328.11651611328125, + "KL/rejected_KL_mean": -412.26641845703125, + "KL/std": 171.32447814941406, + "epoch": 0.39500734214390604, + "fcm_dpo/beta": 0.0031313346698880196, + "fcm_dpo/delta": -0.13035503029823303, + "fcm_dpo/margin": 168.2998504638672, + "fcm_dpo/q_t": 0.38259977102279663, + "grad_norm": 30.891639709472656, + "learning_rate": 3.8051520207480204e-07, + "logits/chosen": -0.4082416594028473, + "logits/rejected": -0.3874931335449219, + "logps/chosen": -314.00616455078125, + "logps/ref_chosen": -70.03955841064453, + "logps/ref_rejected": -107.34937286376953, + "logps/rejected": -519.6158447265625, + "loss": 1.0322, + "margin_dpo/margin_mean": 168.29983520507812, + "margin_dpo/margin_std": 217.715576171875, + "step": 269 + }, + { + "KL/chosen_KL_mean": -209.37474060058594, + "KL/mean": -271.95452880859375, + "KL/rejected_KL_mean": -334.53436279296875, + "KL/std": 150.77197265625, + "epoch": 0.3964757709251101, + "fcm_dpo/beta": 0.0031246920116245747, + "fcm_dpo/delta": 0.008997350931167603, + "fcm_dpo/margin": 125.15963745117188, + "fcm_dpo/q_t": 0.41077637672424316, + "grad_norm": 45.283546447753906, + "learning_rate": 3.794189242333106e-07, + "logits/chosen": -0.4233609139919281, + "logits/rejected": -0.41205936670303345, + "logps/chosen": -278.908203125, + "logps/ref_chosen": -69.53347778320312, + "logps/ref_rejected": -109.92864990234375, + "logps/rejected": -444.4630126953125, + "loss": 1.1118, + "margin_dpo/margin_mean": 125.15963745117188, + "margin_dpo/margin_std": 185.95555114746094, + "step": 270 + }, + { + "KL/chosen_KL_mean": -188.9541473388672, + "KL/mean": -265.62396240234375, + "KL/rejected_KL_mean": -342.2938232421875, + "KL/std": 149.54208374023438, + "epoch": 0.39794419970631423, + "fcm_dpo/beta": 0.0030992082320153713, + "fcm_dpo/delta": -0.07721711695194244, + "fcm_dpo/margin": 153.33963012695312, + "fcm_dpo/q_t": 0.39120548963546753, + "grad_norm": 27.018428802490234, + "learning_rate": 3.7831923608280514e-07, + "logits/chosen": -0.3589969277381897, + "logits/rejected": -0.33432579040527344, + "logps/chosen": -245.71871948242188, + "logps/ref_chosen": -56.76456832885742, + "logps/ref_rejected": -92.51383972167969, + "logps/rejected": -434.8076477050781, + "loss": 1.0279, + "margin_dpo/margin_mean": 153.3396453857422, + "margin_dpo/margin_std": 167.06747436523438, + "step": 271 + }, + { + "KL/chosen_KL_mean": -173.87306213378906, + "KL/mean": -266.062744140625, + "KL/rejected_KL_mean": -358.25244140625, + "KL/std": 168.98089599609375, + "epoch": 0.39941262848751835, + "fcm_dpo/beta": 0.003064734861254692, + "fcm_dpo/delta": -0.16969643533229828, + "fcm_dpo/margin": 184.37937927246094, + "fcm_dpo/q_t": 0.3702283501625061, + "grad_norm": 34.31309509277344, + "learning_rate": 3.772161666010912e-07, + "logits/chosen": -0.31992781162261963, + "logits/rejected": -0.3296176791191101, + "logps/chosen": -223.37022399902344, + "logps/ref_chosen": -49.497154235839844, + "logps/ref_rejected": -105.54279327392578, + "logps/rejected": -463.79522705078125, + "loss": 0.9675, + "margin_dpo/margin_mean": 184.37937927246094, + "margin_dpo/margin_std": 175.53350830078125, + "step": 272 + }, + { + "KL/chosen_KL_mean": -202.81802368164062, + "KL/mean": -292.4516296386719, + "KL/rejected_KL_mean": -382.08526611328125, + "KL/std": 168.8096923828125, + "epoch": 0.4008810572687225, + "fcm_dpo/beta": 0.0030071022920310497, + "fcm_dpo/delta": -0.14330768585205078, + "fcm_dpo/margin": 179.26719665527344, + "fcm_dpo/q_t": 0.37718284130096436, + "grad_norm": 23.817079544067383, + "learning_rate": 3.761097448550755e-07, + "logits/chosen": -0.3086550533771515, + "logits/rejected": -0.28471726179122925, + "logps/chosen": -265.7934265136719, + "logps/ref_chosen": -62.97539520263672, + "logps/ref_rejected": -92.49858093261719, + "logps/rejected": -474.5838317871094, + "loss": 0.9925, + "margin_dpo/margin_mean": 179.26719665527344, + "margin_dpo/margin_std": 188.98374938964844, + "step": 273 + }, + { + "KL/chosen_KL_mean": -239.08636474609375, + "KL/mean": -309.47210693359375, + "KL/rejected_KL_mean": -379.85784912109375, + "KL/std": 157.9844970703125, + "epoch": 0.4023494860499266, + "fcm_dpo/beta": 0.003000047989189625, + "fcm_dpo/delta": -0.02283497154712677, + "fcm_dpo/margin": 140.77145385742188, + "fcm_dpo/q_t": 0.4034059941768646, + "grad_norm": 29.533851623535156, + "learning_rate": 3.75e-07, + "logits/chosen": -0.2807186245918274, + "logits/rejected": -0.260738730430603, + "logps/chosen": -294.75408935546875, + "logps/ref_chosen": -55.66770935058594, + "logps/ref_rejected": -77.33308410644531, + "logps/rejected": -457.19091796875, + "loss": 1.077, + "margin_dpo/margin_mean": 140.77145385742188, + "margin_dpo/margin_std": 181.21035766601562, + "step": 274 + }, + { + "KL/chosen_KL_mean": -190.72259521484375, + "KL/mean": -263.8143005371094, + "KL/rejected_KL_mean": -336.906005859375, + "KL/std": 159.49850463867188, + "epoch": 0.40381791483113066, + "fcm_dpo/beta": 0.002990099135786295, + "fcm_dpo/delta": -0.03795165196061134, + "fcm_dpo/margin": 146.18341064453125, + "fcm_dpo/q_t": 0.39963993430137634, + "grad_norm": 28.2340145111084, + "learning_rate": 3.738869612786737e-07, + "logits/chosen": -0.3173510432243347, + "logits/rejected": -0.3210340142250061, + "logps/chosen": -239.31729125976562, + "logps/ref_chosen": -48.594703674316406, + "logps/ref_rejected": -93.30369567871094, + "logps/rejected": -430.209716796875, + "loss": 1.0664, + "margin_dpo/margin_mean": 146.18341064453125, + "margin_dpo/margin_std": 182.6461181640625, + "step": 275 + }, + { + "KL/chosen_KL_mean": -207.98233032226562, + "KL/mean": -282.283203125, + "KL/rejected_KL_mean": -356.5841064453125, + "KL/std": 165.94070434570312, + "epoch": 0.4052863436123348, + "fcm_dpo/beta": 0.002971104346215725, + "fcm_dpo/delta": -0.04252216964960098, + "fcm_dpo/margin": 148.60177612304688, + "fcm_dpo/q_t": 0.4004755914211273, + "grad_norm": 30.543088912963867, + "learning_rate": 3.7277065802070204e-07, + "logits/chosen": -0.31873783469200134, + "logits/rejected": -0.29504579305648804, + "logps/chosen": -264.55975341796875, + "logps/ref_chosen": -56.57740783691406, + "logps/ref_rejected": -70.36566925048828, + "logps/rejected": -426.94976806640625, + "loss": 1.0728, + "margin_dpo/margin_mean": 148.60177612304688, + "margin_dpo/margin_std": 199.07870483398438, + "step": 276 + }, + { + "KL/chosen_KL_mean": -230.2869110107422, + "KL/mean": -309.2855529785156, + "KL/rejected_KL_mean": -388.2841796875, + "KL/std": 167.18478393554688, + "epoch": 0.4067547723935389, + "fcm_dpo/beta": 0.0029588094912469387, + "fcm_dpo/delta": -0.0690733790397644, + "fcm_dpo/margin": 157.99728393554688, + "fcm_dpo/q_t": 0.39369434118270874, + "grad_norm": 26.234636306762695, + "learning_rate": 3.71651119641714e-07, + "logits/chosen": -0.29924240708351135, + "logits/rejected": -0.2815479636192322, + "logps/chosen": -286.5584716796875, + "logps/ref_chosen": -56.27156066894531, + "logps/ref_rejected": -92.88127136230469, + "logps/rejected": -481.16546630859375, + "loss": 1.0493, + "margin_dpo/margin_mean": 157.99728393554688, + "margin_dpo/margin_std": 193.92010498046875, + "step": 277 + }, + { + "KL/chosen_KL_mean": -206.88705444335938, + "KL/mean": -296.92340087890625, + "KL/rejected_KL_mean": -386.95977783203125, + "KL/std": 179.54661560058594, + "epoch": 0.40822320117474303, + "fcm_dpo/beta": 0.0029181232675909996, + "fcm_dpo/delta": -0.12909561395645142, + "fcm_dpo/margin": 180.07273864746094, + "fcm_dpo/q_t": 0.38142508268356323, + "grad_norm": 31.345380783081055, + "learning_rate": 3.705283756425872e-07, + "logits/chosen": -0.2834761142730713, + "logits/rejected": -0.28856098651885986, + "logps/chosen": -259.8289794921875, + "logps/ref_chosen": -52.94194030761719, + "logps/ref_rejected": -91.25357818603516, + "logps/rejected": -478.21337890625, + "loss": 1.004, + "margin_dpo/margin_mean": 180.07275390625, + "margin_dpo/margin_std": 198.38058471679688, + "step": 278 + }, + { + "KL/chosen_KL_mean": -245.1502227783203, + "KL/mean": -334.9271240234375, + "KL/rejected_KL_mean": -424.7039794921875, + "KL/std": 189.08995056152344, + "epoch": 0.40969162995594716, + "fcm_dpo/beta": 0.0028769150376319885, + "fcm_dpo/delta": -0.12047116458415985, + "fcm_dpo/margin": 179.55380249023438, + "fcm_dpo/q_t": 0.38520491123199463, + "grad_norm": 30.17645263671875, + "learning_rate": 3.6940245560867e-07, + "logits/chosen": -0.23838657140731812, + "logits/rejected": -0.23660916090011597, + "logps/chosen": -293.7915344238281, + "logps/ref_chosen": -48.641319274902344, + "logps/ref_rejected": -87.8514404296875, + "logps/rejected": -512.555419921875, + "loss": 1.0304, + "margin_dpo/margin_mean": 179.55380249023438, + "margin_dpo/margin_std": 223.5946044921875, + "step": 279 + }, + { + "KL/chosen_KL_mean": -248.6533203125, + "KL/mean": -343.9190673828125, + "KL/rejected_KL_mean": -439.184814453125, + "KL/std": 181.25201416015625, + "epoch": 0.4111600587371512, + "fcm_dpo/beta": 0.002845579292625189, + "fcm_dpo/delta": -0.14617213606834412, + "fcm_dpo/margin": 190.531494140625, + "fcm_dpo/q_t": 0.3750941753387451, + "grad_norm": 27.4671688079834, + "learning_rate": 3.6827338920900253e-07, + "logits/chosen": -0.271923303604126, + "logits/rejected": -0.2730262279510498, + "logps/chosen": -307.450439453125, + "logps/ref_chosen": -58.797122955322266, + "logps/ref_rejected": -98.61885070800781, + "logps/rejected": -537.8037109375, + "loss": 0.9844, + "margin_dpo/margin_mean": 190.53150939941406, + "margin_dpo/margin_std": 191.6612548828125, + "step": 280 + }, + { + "KL/chosen_KL_mean": -226.9546356201172, + "KL/mean": -309.49078369140625, + "KL/rejected_KL_mean": -392.0268859863281, + "KL/std": 174.78067016601562, + "epoch": 0.41262848751835535, + "fcm_dpo/beta": 0.0028265006840229034, + "fcm_dpo/delta": -0.06817743927240372, + "fcm_dpo/margin": 165.07223510742188, + "fcm_dpo/q_t": 0.39088958501815796, + "grad_norm": 26.444442749023438, + "learning_rate": 3.6714120619553435e-07, + "logits/chosen": -0.3203830122947693, + "logits/rejected": -0.2967671751976013, + "logps/chosen": -282.44317626953125, + "logps/ref_chosen": -55.488521575927734, + "logps/ref_rejected": -80.88258361816406, + "logps/rejected": -472.9094543457031, + "loss": 1.0406, + "margin_dpo/margin_mean": 165.07223510742188, + "margin_dpo/margin_std": 188.06488037109375, + "step": 281 + }, + { + "KL/chosen_KL_mean": -251.03106689453125, + "KL/mean": -316.7037658691406, + "KL/rejected_KL_mean": -382.37646484375, + "KL/std": 178.154541015625, + "epoch": 0.41409691629955947, + "fcm_dpo/beta": 0.0028289747424423695, + "fcm_dpo/delta": 0.028688903898000717, + "fcm_dpo/margin": 131.34539794921875, + "fcm_dpo/q_t": 0.41740673780441284, + "grad_norm": 27.60743522644043, + "learning_rate": 3.660059364023408e-07, + "logits/chosen": -0.3705713748931885, + "logits/rejected": -0.34871238470077515, + "logps/chosen": -324.1011962890625, + "logps/ref_chosen": -73.07014465332031, + "logps/ref_rejected": -95.35098266601562, + "logps/rejected": -477.7274169921875, + "loss": 1.1241, + "margin_dpo/margin_mean": 131.34539794921875, + "margin_dpo/margin_std": 205.223876953125, + "step": 282 + }, + { + "KL/chosen_KL_mean": -265.2757263183594, + "KL/mean": -361.2529296875, + "KL/rejected_KL_mean": -457.23016357421875, + "KL/std": 214.1881103515625, + "epoch": 0.4155653450807636, + "fcm_dpo/beta": 0.002800577785819769, + "fcm_dpo/delta": -0.14126329123973846, + "fcm_dpo/margin": 191.95440673828125, + "fcm_dpo/q_t": 0.37666594982147217, + "grad_norm": 34.12551498413086, + "learning_rate": 3.6486760974483685e-07, + "logits/chosen": -0.3212432265281677, + "logits/rejected": -0.32373398542404175, + "logps/chosen": -327.1741638183594, + "logps/ref_chosen": -61.89844512939453, + "logps/ref_rejected": -96.98655700683594, + "logps/rejected": -554.2166748046875, + "loss": 0.9907, + "margin_dpo/margin_mean": 191.9544219970703, + "margin_dpo/margin_std": 196.35537719726562, + "step": 283 + }, + { + "KL/chosen_KL_mean": -247.26083374023438, + "KL/mean": -337.656982421875, + "KL/rejected_KL_mean": -428.0531005859375, + "KL/std": 198.7021484375, + "epoch": 0.4170337738619677, + "fcm_dpo/beta": 0.0027607777155935764, + "fcm_dpo/delta": -0.10182726383209229, + "fcm_dpo/margin": 180.79226684570312, + "fcm_dpo/q_t": 0.38608792424201965, + "grad_norm": 41.67982864379883, + "learning_rate": 3.6372625621898863e-07, + "logits/chosen": -0.3618922829627991, + "logits/rejected": -0.3465798795223236, + "logps/chosen": -305.6963806152344, + "logps/ref_chosen": -58.4355354309082, + "logps/ref_rejected": -93.46926879882812, + "logps/rejected": -521.5223999023438, + "loss": 1.0107, + "margin_dpo/margin_mean": 180.79226684570312, + "margin_dpo/margin_std": 191.53631591796875, + "step": 284 + }, + { + "KL/chosen_KL_mean": -280.42071533203125, + "KL/mean": -361.66033935546875, + "KL/rejected_KL_mean": -442.8999938964844, + "KL/std": 175.34622192382812, + "epoch": 0.4185022026431718, + "fcm_dpo/beta": 0.002754541113972664, + "fcm_dpo/delta": -0.048833686858415604, + "fcm_dpo/margin": 162.47930908203125, + "fcm_dpo/q_t": 0.3953213095664978, + "grad_norm": 26.926042556762695, + "learning_rate": 3.625819059005228e-07, + "logits/chosen": -0.3326127529144287, + "logits/rejected": -0.31620490550994873, + "logps/chosen": -346.65289306640625, + "logps/ref_chosen": -66.23219299316406, + "logps/ref_rejected": -99.1268310546875, + "logps/rejected": -542.02685546875, + "loss": 1.0516, + "margin_dpo/margin_mean": 162.47930908203125, + "margin_dpo/margin_std": 184.34237670898438, + "step": 285 + }, + { + "KL/chosen_KL_mean": -277.4112548828125, + "KL/mean": -369.2188720703125, + "KL/rejected_KL_mean": -461.0264892578125, + "KL/std": 199.40582275390625, + "epoch": 0.4199706314243759, + "fcm_dpo/beta": 0.0027244749944657087, + "fcm_dpo/delta": -0.10277135670185089, + "fcm_dpo/margin": 183.615234375, + "fcm_dpo/q_t": 0.38644248247146606, + "grad_norm": 28.425804138183594, + "learning_rate": 3.614345889441346e-07, + "logits/chosen": -0.3559093475341797, + "logits/rejected": -0.3347648084163666, + "logps/chosen": -350.36224365234375, + "logps/ref_chosen": -72.95100402832031, + "logps/ref_rejected": -88.58845520019531, + "logps/rejected": -549.6149291992188, + "loss": 1.0286, + "margin_dpo/margin_mean": 183.615234375, + "margin_dpo/margin_std": 217.96173095703125, + "step": 286 + }, + { + "KL/chosen_KL_mean": -261.5997314453125, + "KL/mean": -335.04229736328125, + "KL/rejected_KL_mean": -408.4848327636719, + "KL/std": 173.95217895507812, + "epoch": 0.42143906020558003, + "fcm_dpo/beta": 0.002716212999075651, + "fcm_dpo/delta": 0.0010463353246450424, + "fcm_dpo/margin": 146.88510131835938, + "fcm_dpo/q_t": 0.4084421694278717, + "grad_norm": 26.484556198120117, + "learning_rate": 3.6028433558269275e-07, + "logits/chosen": -0.3276909589767456, + "logits/rejected": -0.30225396156311035, + "logps/chosen": -323.140869140625, + "logps/ref_chosen": -61.54115295410156, + "logps/ref_rejected": -77.69607543945312, + "logps/rejected": -486.180908203125, + "loss": 1.0875, + "margin_dpo/margin_mean": 146.88510131835938, + "margin_dpo/margin_std": 189.57159423828125, + "step": 287 + }, + { + "KL/chosen_KL_mean": -250.09185791015625, + "KL/mean": -342.5625, + "KL/rejected_KL_mean": -435.0331726074219, + "KL/std": 174.55943298339844, + "epoch": 0.42290748898678415, + "fcm_dpo/beta": 0.0026842262595891953, + "fcm_dpo/delta": -0.09998185932636261, + "fcm_dpo/margin": 184.9413299560547, + "fcm_dpo/q_t": 0.38463830947875977, + "grad_norm": 21.34337043762207, + "learning_rate": 3.5913117612644327e-07, + "logits/chosen": -0.3534676432609558, + "logits/rejected": -0.3342832326889038, + "logps/chosen": -306.7530517578125, + "logps/ref_chosen": -56.661224365234375, + "logps/ref_rejected": -87.33570098876953, + "logps/rejected": -522.368896484375, + "loss": 1.0087, + "margin_dpo/margin_mean": 184.94134521484375, + "margin_dpo/margin_std": 177.844970703125, + "step": 288 + }, + { + "KL/chosen_KL_mean": -247.0579833984375, + "KL/mean": -350.28240966796875, + "KL/rejected_KL_mean": -453.50677490234375, + "KL/std": 198.4517822265625, + "epoch": 0.4243759177679883, + "fcm_dpo/beta": 0.002661726903170347, + "fcm_dpo/delta": -0.15359723567962646, + "fcm_dpo/margin": 206.44882202148438, + "fcm_dpo/q_t": 0.3754338324069977, + "grad_norm": 26.822708129882812, + "learning_rate": 3.5797514096221024e-07, + "logits/chosen": -0.2866683602333069, + "logits/rejected": -0.28309381008148193, + "logps/chosen": -292.28839111328125, + "logps/ref_chosen": -45.23039245605469, + "logps/ref_rejected": -87.64266967773438, + "logps/rejected": -541.1494140625, + "loss": 0.9846, + "margin_dpo/margin_mean": 206.44879150390625, + "margin_dpo/margin_std": 213.51644897460938, + "step": 289 + }, + { + "KL/chosen_KL_mean": -268.5120849609375, + "KL/mean": -371.7317810058594, + "KL/rejected_KL_mean": -474.95147705078125, + "KL/std": 210.44107055664062, + "epoch": 0.42584434654919234, + "fcm_dpo/beta": 0.002616926096379757, + "fcm_dpo/delta": -0.14428411424160004, + "fcm_dpo/margin": 206.43942260742188, + "fcm_dpo/q_t": 0.3804370164871216, + "grad_norm": 24.522186279296875, + "learning_rate": 3.568162605525952e-07, + "logits/chosen": -0.2609245181083679, + "logits/rejected": -0.2813086211681366, + "logps/chosen": -323.98358154296875, + "logps/ref_chosen": -55.47149658203125, + "logps/ref_rejected": -116.70857238769531, + "logps/rejected": -591.6600341796875, + "loss": 1.0094, + "margin_dpo/margin_mean": 206.43942260742188, + "margin_dpo/margin_std": 247.92044067382812, + "step": 290 + }, + { + "KL/chosen_KL_mean": -230.06326293945312, + "KL/mean": -323.6925048828125, + "KL/rejected_KL_mean": -417.32177734375, + "KL/std": 181.36814880371094, + "epoch": 0.42731277533039647, + "fcm_dpo/beta": 0.0025994423776865005, + "fcm_dpo/delta": -0.08899101614952087, + "fcm_dpo/margin": 187.25848388671875, + "fcm_dpo/q_t": 0.3880588114261627, + "grad_norm": 31.68715476989746, + "learning_rate": 3.5565456543517485e-07, + "logits/chosen": -0.3303547501564026, + "logits/rejected": -0.31436848640441895, + "logps/chosen": -293.3236083984375, + "logps/ref_chosen": -63.26036834716797, + "logps/ref_rejected": -89.29708862304688, + "logps/rejected": -506.61883544921875, + "loss": 1.0307, + "margin_dpo/margin_mean": 187.25848388671875, + "margin_dpo/margin_std": 211.5198974609375, + "step": 291 + }, + { + "KL/chosen_KL_mean": -261.43878173828125, + "KL/mean": -362.51861572265625, + "KL/rejected_KL_mean": -463.598388671875, + "KL/std": 217.88809204101562, + "epoch": 0.4287812041116006, + "fcm_dpo/beta": 0.0025680752005428076, + "fcm_dpo/delta": -0.12225315719842911, + "fcm_dpo/margin": 202.15960693359375, + "fcm_dpo/q_t": 0.382857084274292, + "grad_norm": 29.966228485107422, + "learning_rate": 3.5449008622169583e-07, + "logits/chosen": -0.3206707537174225, + "logits/rejected": -0.3061414361000061, + "logps/chosen": -315.3572998046875, + "logps/ref_chosen": -53.91852951049805, + "logps/ref_rejected": -89.96138000488281, + "logps/rejected": -553.559814453125, + "loss": 1.0308, + "margin_dpo/margin_mean": 202.15960693359375, + "margin_dpo/margin_std": 255.03884887695312, + "step": 292 + }, + { + "KL/chosen_KL_mean": -266.5294189453125, + "KL/mean": -346.7402038574219, + "KL/rejected_KL_mean": -426.95098876953125, + "KL/std": 211.74429321289062, + "epoch": 0.4302496328928047, + "fcm_dpo/beta": 0.002555608283728361, + "fcm_dpo/delta": -0.010212033987045288, + "fcm_dpo/margin": 160.42156982421875, + "fcm_dpo/q_t": 0.40869617462158203, + "grad_norm": 45.290653228759766, + "learning_rate": 3.5332285359726846e-07, + "logits/chosen": -0.3368123769760132, + "logits/rejected": -0.32695645093917847, + "logps/chosen": -326.90545654296875, + "logps/ref_chosen": -60.376033782958984, + "logps/ref_rejected": -77.85244750976562, + "logps/rejected": -504.8034362792969, + "loss": 1.1009, + "margin_dpo/margin_mean": 160.4215850830078, + "margin_dpo/margin_std": 237.15798950195312, + "step": 293 + }, + { + "KL/chosen_KL_mean": -252.91036987304688, + "KL/mean": -340.8631591796875, + "KL/rejected_KL_mean": -428.81591796875, + "KL/std": 192.0465087890625, + "epoch": 0.43171806167400884, + "fcm_dpo/beta": 0.0025399108417332172, + "fcm_dpo/delta": -0.047990452498197556, + "fcm_dpo/margin": 175.9055633544922, + "fcm_dpo/q_t": 0.400339812040329, + "grad_norm": 29.165407180786133, + "learning_rate": 3.5215289831955786e-07, + "logits/chosen": -0.3062417805194855, + "logits/rejected": -0.3089578151702881, + "logps/chosen": -300.9979248046875, + "logps/ref_chosen": -48.0875358581543, + "logps/ref_rejected": -81.89698791503906, + "logps/rejected": -510.7129211425781, + "loss": 1.0685, + "margin_dpo/margin_mean": 175.9055633544922, + "margin_dpo/margin_std": 233.61227416992188, + "step": 294 + }, + { + "KL/chosen_KL_mean": -301.30224609375, + "KL/mean": -397.0389404296875, + "KL/rejected_KL_mean": -492.775634765625, + "KL/std": 212.322265625, + "epoch": 0.4331864904552129, + "fcm_dpo/beta": 0.002525617368519306, + "fcm_dpo/delta": -0.08562206476926804, + "fcm_dpo/margin": 191.47335815429688, + "fcm_dpo/q_t": 0.3909651041030884, + "grad_norm": 27.463563919067383, + "learning_rate": 3.509802512179737e-07, + "logits/chosen": -0.2896770238876343, + "logits/rejected": -0.290955126285553, + "logps/chosen": -351.2268981933594, + "logps/ref_chosen": -49.92467498779297, + "logps/ref_rejected": -87.45632934570312, + "logps/rejected": -580.23193359375, + "loss": 1.0469, + "margin_dpo/margin_mean": 191.47335815429688, + "margin_dpo/margin_std": 241.90957641601562, + "step": 295 + }, + { + "KL/chosen_KL_mean": -359.9180603027344, + "KL/mean": -426.7522277832031, + "KL/rejected_KL_mean": -493.5863952636719, + "KL/std": 202.1461181640625, + "epoch": 0.434654919236417, + "fcm_dpo/beta": 0.002509201178327203, + "fcm_dpo/delta": -0.03729523718357086, + "fcm_dpo/margin": 133.6683349609375, + "fcm_dpo/q_t": 0.4235140383243561, + "grad_norm": 45.111167907714844, + "learning_rate": 3.498049431928577e-07, + "logits/chosen": -0.3604152202606201, + "logits/rejected": -0.3418993353843689, + "logps/chosen": -425.4093017578125, + "logps/ref_chosen": -65.49124145507812, + "logps/ref_rejected": -93.08908081054688, + "logps/rejected": -586.6754760742188, + "loss": 1.1758, + "margin_dpo/margin_mean": 133.6683349609375, + "margin_dpo/margin_std": 252.33218383789062, + "step": 296 + }, + { + "KL/chosen_KL_mean": -313.9520568847656, + "KL/mean": -391.02191162109375, + "KL/rejected_KL_mean": -468.09173583984375, + "KL/std": 195.4499969482422, + "epoch": 0.43612334801762115, + "fcm_dpo/beta": 0.002508984412997961, + "fcm_dpo/delta": 0.013493766076862812, + "fcm_dpo/margin": 154.13967895507812, + "fcm_dpo/q_t": 0.41029807925224304, + "grad_norm": 49.82147979736328, + "learning_rate": 3.486270052146694e-07, + "logits/chosen": -0.3424445390701294, + "logits/rejected": -0.3403598368167877, + "logps/chosen": -370.42901611328125, + "logps/ref_chosen": -56.476951599121094, + "logps/ref_rejected": -95.1385498046875, + "logps/rejected": -563.2302856445312, + "loss": 1.0935, + "margin_dpo/margin_mean": 154.13967895507812, + "margin_dpo/margin_std": 194.82879638671875, + "step": 297 + }, + { + "KL/chosen_KL_mean": -310.0024719238281, + "KL/mean": -410.1289978027344, + "KL/rejected_KL_mean": -510.2554931640625, + "KL/std": 235.77732849121094, + "epoch": 0.43759177679882527, + "fcm_dpo/beta": 0.0024994537234306335, + "fcm_dpo/delta": -0.10309357941150665, + "fcm_dpo/margin": 200.2530517578125, + "fcm_dpo/q_t": 0.3903202712535858, + "grad_norm": 27.238431930541992, + "learning_rate": 3.474464683231698e-07, + "logits/chosen": -0.3699042797088623, + "logits/rejected": -0.3843574523925781, + "logps/chosen": -377.32763671875, + "logps/ref_chosen": -67.32516479492188, + "logps/ref_rejected": -116.66217041015625, + "logps/rejected": -626.917724609375, + "loss": 1.0457, + "margin_dpo/margin_mean": 200.25302124023438, + "margin_dpo/margin_std": 266.128662109375, + "step": 298 + }, + { + "KL/chosen_KL_mean": -259.36639404296875, + "KL/mean": -347.03094482421875, + "KL/rejected_KL_mean": -434.695556640625, + "KL/std": 185.60708618164062, + "epoch": 0.4390602055800294, + "fcm_dpo/beta": 0.0024865521118044853, + "fcm_dpo/delta": -0.03695205599069595, + "fcm_dpo/margin": 175.32911682128906, + "fcm_dpo/q_t": 0.40157145261764526, + "grad_norm": 34.25696563720703, + "learning_rate": 3.462633636266041e-07, + "logits/chosen": -0.36456483602523804, + "logits/rejected": -0.36391931772232056, + "logps/chosen": -308.3284912109375, + "logps/ref_chosen": -48.96209716796875, + "logps/ref_rejected": -84.32823944091797, + "logps/rejected": -519.0238037109375, + "loss": 1.0759, + "margin_dpo/margin_mean": 175.3291015625, + "margin_dpo/margin_std": 231.766357421875, + "step": 299 + }, + { + "KL/chosen_KL_mean": -315.7066650390625, + "KL/mean": -422.8812255859375, + "KL/rejected_KL_mean": -530.0557861328125, + "KL/std": 226.07566833496094, + "epoch": 0.44052863436123346, + "fcm_dpo/beta": 0.0024577926378697157, + "fcm_dpo/delta": -0.1301524043083191, + "fcm_dpo/margin": 214.34909057617188, + "fcm_dpo/q_t": 0.3810431957244873, + "grad_norm": 35.657649993896484, + "learning_rate": 3.4507772230088147e-07, + "logits/chosen": -0.35023725032806396, + "logits/rejected": -0.34927883744239807, + "logps/chosen": -374.7803955078125, + "logps/ref_chosen": -59.07371139526367, + "logps/ref_rejected": -95.9664535522461, + "logps/rejected": -626.022216796875, + "loss": 1.0246, + "margin_dpo/margin_mean": 214.34910583496094, + "margin_dpo/margin_std": 262.06231689453125, + "step": 300 + }, + { + "KL/chosen_KL_mean": -261.52374267578125, + "KL/mean": -361.13104248046875, + "KL/rejected_KL_mean": -460.73834228515625, + "KL/std": 200.12277221679688, + "epoch": 0.4419970631424376, + "fcm_dpo/beta": 0.002426966093480587, + "fcm_dpo/delta": -0.0857255756855011, + "fcm_dpo/margin": 199.21456909179688, + "fcm_dpo/q_t": 0.3913128077983856, + "grad_norm": 29.40239906311035, + "learning_rate": 3.4388957558875316e-07, + "logits/chosen": -0.38150107860565186, + "logits/rejected": -0.3794858753681183, + "logps/chosen": -318.77313232421875, + "logps/ref_chosen": -57.249366760253906, + "logps/ref_rejected": -92.35354614257812, + "logps/rejected": -553.0919189453125, + "loss": 1.0345, + "margin_dpo/margin_mean": 199.21456909179688, + "margin_dpo/margin_std": 237.46759033203125, + "step": 301 + }, + { + "KL/chosen_KL_mean": -216.02481079101562, + "KL/mean": -303.01495361328125, + "KL/rejected_KL_mean": -390.005126953125, + "KL/std": 180.0056915283203, + "epoch": 0.4434654919236417, + "fcm_dpo/beta": 0.0024160263128578663, + "fcm_dpo/delta": -0.020986871793866158, + "fcm_dpo/margin": 173.98028564453125, + "fcm_dpo/q_t": 0.4036945402622223, + "grad_norm": 19.06124496459961, + "learning_rate": 3.426989547989902e-07, + "logits/chosen": -0.3539729118347168, + "logits/rejected": -0.3578333854675293, + "logps/chosen": -267.2228088378906, + "logps/ref_chosen": -51.197994232177734, + "logps/ref_rejected": -97.22636413574219, + "logps/rejected": -487.2314758300781, + "loss": 1.0715, + "margin_dpo/margin_mean": 173.98028564453125, + "margin_dpo/margin_std": 212.90484619140625, + "step": 302 + }, + { + "KL/chosen_KL_mean": -238.58770751953125, + "KL/mean": -316.30780029296875, + "KL/rejected_KL_mean": -394.02789306640625, + "KL/std": 194.1252899169922, + "epoch": 0.44493392070484583, + "fcm_dpo/beta": 0.002420827280730009, + "fcm_dpo/delta": 0.024122366681694984, + "fcm_dpo/margin": 155.44015502929688, + "fcm_dpo/q_t": 0.4143924117088318, + "grad_norm": 24.42957305908203, + "learning_rate": 3.4150589130555773e-07, + "logits/chosen": -0.37989115715026855, + "logits/rejected": -0.3611317574977875, + "logps/chosen": -305.3016357421875, + "logps/ref_chosen": -66.71394348144531, + "logps/ref_rejected": -86.94542694091797, + "logps/rejected": -480.97332763671875, + "loss": 1.1184, + "margin_dpo/margin_mean": 155.440185546875, + "margin_dpo/margin_std": 229.31715393066406, + "step": 303 + }, + { + "KL/chosen_KL_mean": -208.25198364257812, + "KL/mean": -296.2002258300781, + "KL/rejected_KL_mean": -384.1484375, + "KL/std": 177.62567138671875, + "epoch": 0.44640234948604995, + "fcm_dpo/beta": 0.002425071783363819, + "fcm_dpo/delta": -0.027229398488998413, + "fcm_dpo/margin": 175.89645385742188, + "fcm_dpo/q_t": 0.39837926626205444, + "grad_norm": 29.601119995117188, + "learning_rate": 3.403104165467883e-07, + "logits/chosen": -0.3989083766937256, + "logits/rejected": -0.3849946856498718, + "logps/chosen": -280.20269775390625, + "logps/ref_chosen": -71.95069885253906, + "logps/ref_rejected": -90.47203063964844, + "logps/rejected": -474.6204833984375, + "loss": 1.0409, + "margin_dpo/margin_mean": 175.89645385742188, + "margin_dpo/margin_std": 156.81117248535156, + "step": 304 + }, + { + "KL/chosen_KL_mean": -231.21951293945312, + "KL/mean": -307.9289855957031, + "KL/rejected_KL_mean": -384.6384582519531, + "KL/std": 201.5638885498047, + "epoch": 0.447870778267254, + "fcm_dpo/beta": 0.002417425625026226, + "fcm_dpo/delta": 0.02943410724401474, + "fcm_dpo/margin": 153.41891479492188, + "fcm_dpo/q_t": 0.41627079248428345, + "grad_norm": 27.138019561767578, + "learning_rate": 3.391125620245535e-07, + "logits/chosen": -0.41401392221450806, + "logits/rejected": -0.39217787981033325, + "logps/chosen": -298.0147705078125, + "logps/ref_chosen": -66.79523468017578, + "logps/ref_rejected": -92.75459289550781, + "logps/rejected": -477.39306640625, + "loss": 1.1181, + "margin_dpo/margin_mean": 153.41891479492188, + "margin_dpo/margin_std": 221.77337646484375, + "step": 305 + }, + { + "KL/chosen_KL_mean": -229.625244140625, + "KL/mean": -305.220947265625, + "KL/rejected_KL_mean": -380.8166198730469, + "KL/std": 177.791015625, + "epoch": 0.44933920704845814, + "fcm_dpo/beta": 0.0024337819777429104, + "fcm_dpo/delta": 0.03258597105741501, + "fcm_dpo/margin": 151.19137573242188, + "fcm_dpo/q_t": 0.4141908884048462, + "grad_norm": 26.584671020507812, + "learning_rate": 3.3791235930343417e-07, + "logits/chosen": -0.35081660747528076, + "logits/rejected": -0.3195233941078186, + "logps/chosen": -299.30914306640625, + "logps/ref_chosen": -69.68389892578125, + "logps/ref_rejected": -85.15919494628906, + "logps/rejected": -465.9757995605469, + "loss": 1.0999, + "margin_dpo/margin_mean": 151.19139099121094, + "margin_dpo/margin_std": 187.10317993164062, + "step": 306 + }, + { + "KL/chosen_KL_mean": -215.2943572998047, + "KL/mean": -296.1317138671875, + "KL/rejected_KL_mean": -376.96905517578125, + "KL/std": 165.89390563964844, + "epoch": 0.45080763582966227, + "fcm_dpo/beta": 0.0024325139820575714, + "fcm_dpo/delta": 0.00681762769818306, + "fcm_dpo/margin": 161.6746826171875, + "fcm_dpo/q_t": 0.4085942506790161, + "grad_norm": 25.906414031982422, + "learning_rate": 3.367098400098881e-07, + "logits/chosen": -0.3687888979911804, + "logits/rejected": -0.34732580184936523, + "logps/chosen": -285.45977783203125, + "logps/ref_chosen": -70.16542053222656, + "logps/ref_rejected": -86.97230529785156, + "logps/rejected": -463.94134521484375, + "loss": 1.0872, + "margin_dpo/margin_mean": 161.6746826171875, + "margin_dpo/margin_std": 200.76708984375, + "step": 307 + }, + { + "KL/chosen_KL_mean": -225.435546875, + "KL/mean": -316.33099365234375, + "KL/rejected_KL_mean": -407.2264404296875, + "KL/std": 194.58316040039062, + "epoch": 0.4522760646108664, + "fcm_dpo/beta": 0.0024283677339553833, + "fcm_dpo/delta": -0.04238416999578476, + "fcm_dpo/margin": 181.79086303710938, + "fcm_dpo/q_t": 0.3963702917098999, + "grad_norm": 29.603750228881836, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": -0.34480029344558716, + "logits/rejected": -0.3308371305465698, + "logps/chosen": -280.6805419921875, + "logps/ref_chosen": -55.2449951171875, + "logps/ref_rejected": -79.37226104736328, + "logps/rejected": -486.59869384765625, + "loss": 1.0408, + "margin_dpo/margin_mean": 181.79086303710938, + "margin_dpo/margin_std": 184.86737060546875, + "step": 308 + }, + { + "KL/chosen_KL_mean": -229.71560668945312, + "KL/mean": -321.12689208984375, + "KL/rejected_KL_mean": -412.5382080078125, + "KL/std": 200.42385864257812, + "epoch": 0.45374449339207046, + "fcm_dpo/beta": 0.0024233111180365086, + "fcm_dpo/delta": -0.04414871707558632, + "fcm_dpo/margin": 182.82260131835938, + "fcm_dpo/q_t": 0.3967331647872925, + "grad_norm": 40.203651428222656, + "learning_rate": 3.3429797851573183e-07, + "logits/chosen": -0.30514609813690186, + "logits/rejected": -0.2906920909881592, + "logps/chosen": -278.6746826171875, + "logps/ref_chosen": -48.959083557128906, + "logps/ref_rejected": -82.34072875976562, + "logps/rejected": -494.8789367675781, + "loss": 1.0527, + "margin_dpo/margin_mean": 182.82260131835938, + "margin_dpo/margin_std": 206.30902099609375, + "step": 309 + }, + { + "KL/chosen_KL_mean": -269.08026123046875, + "KL/mean": -356.98431396484375, + "KL/rejected_KL_mean": -444.88836669921875, + "KL/std": 182.12594604492188, + "epoch": 0.4552129221732746, + "fcm_dpo/beta": 0.00241499999538064, + "fcm_dpo/delta": -0.02522626519203186, + "fcm_dpo/margin": 175.80810546875, + "fcm_dpo/q_t": 0.3999808430671692, + "grad_norm": 25.993465423583984, + "learning_rate": 3.3308869986991487e-07, + "logits/chosen": -0.37713727355003357, + "logits/rejected": -0.3597584366798401, + "logps/chosen": -331.8220520019531, + "logps/ref_chosen": -62.74177932739258, + "logps/ref_rejected": -79.9302978515625, + "logps/rejected": -524.8186645507812, + "loss": 1.0501, + "margin_dpo/margin_mean": 175.80810546875, + "margin_dpo/margin_std": 173.99118041992188, + "step": 310 + }, + { + "KL/chosen_KL_mean": -291.9866943359375, + "KL/mean": -389.12078857421875, + "KL/rejected_KL_mean": -486.2549133300781, + "KL/std": 237.48434448242188, + "epoch": 0.4566813509544787, + "fcm_dpo/beta": 0.0023961286060512066, + "fcm_dpo/delta": -0.06706373393535614, + "fcm_dpo/margin": 194.26821899414062, + "fcm_dpo/q_t": 0.39537882804870605, + "grad_norm": 25.610626220703125, + "learning_rate": 3.3187723175958346e-07, + "logits/chosen": -0.33371835947036743, + "logits/rejected": -0.3037863075733185, + "logps/chosen": -345.0146484375, + "logps/ref_chosen": -53.02798080444336, + "logps/ref_rejected": -77.43820190429688, + "logps/rejected": -563.693115234375, + "loss": 1.0552, + "margin_dpo/margin_mean": 194.26821899414062, + "margin_dpo/margin_std": 249.4111328125, + "step": 311 + }, + { + "KL/chosen_KL_mean": -284.6394348144531, + "KL/mean": -373.61962890625, + "KL/rejected_KL_mean": -462.599853515625, + "KL/std": 207.23092651367188, + "epoch": 0.4581497797356828, + "fcm_dpo/beta": 0.002392760245129466, + "fcm_dpo/delta": -0.0265023373067379, + "fcm_dpo/margin": 177.96038818359375, + "fcm_dpo/q_t": 0.40469247102737427, + "grad_norm": 26.777978897094727, + "learning_rate": 3.306636061080487e-07, + "logits/chosen": -0.29555267095565796, + "logits/rejected": -0.2810285985469818, + "logps/chosen": -334.0316467285156, + "logps/ref_chosen": -49.39221954345703, + "logps/ref_rejected": -75.79280853271484, + "logps/rejected": -538.3926391601562, + "loss": 1.0824, + "margin_dpo/margin_mean": 177.96038818359375, + "margin_dpo/margin_std": 244.01686096191406, + "step": 312 + }, + { + "KL/chosen_KL_mean": -274.753662109375, + "KL/mean": -365.9559326171875, + "KL/rejected_KL_mean": -457.1582336425781, + "KL/std": 222.33108520507812, + "epoch": 0.45961820851688695, + "fcm_dpo/beta": 0.0023880554363131523, + "fcm_dpo/delta": -0.03676465526223183, + "fcm_dpo/margin": 182.40457153320312, + "fcm_dpo/q_t": 0.4009496569633484, + "grad_norm": 25.21873664855957, + "learning_rate": 3.2944785489547537e-07, + "logits/chosen": -0.35764968395233154, + "logits/rejected": -0.3531670570373535, + "logps/chosen": -324.90643310546875, + "logps/ref_chosen": -50.152740478515625, + "logps/ref_rejected": -86.40620422363281, + "logps/rejected": -543.564453125, + "loss": 1.0789, + "margin_dpo/margin_mean": 182.40457153320312, + "margin_dpo/margin_std": 242.63485717773438, + "step": 313 + }, + { + "KL/chosen_KL_mean": -293.580810546875, + "KL/mean": -380.2122802734375, + "KL/rejected_KL_mean": -466.84375, + "KL/std": 214.81533813476562, + "epoch": 0.461086637298091, + "fcm_dpo/beta": 0.0023697358556091785, + "fcm_dpo/delta": -0.011065401136875153, + "fcm_dpo/margin": 173.26296997070312, + "fcm_dpo/q_t": 0.4087506830692291, + "grad_norm": 21.553348541259766, + "learning_rate": 3.2823001015803857e-07, + "logits/chosen": -0.3941684067249298, + "logits/rejected": -0.39253175258636475, + "logps/chosen": -350.818359375, + "logps/ref_chosen": -57.237579345703125, + "logps/ref_rejected": -97.5965347290039, + "logps/rejected": -564.4403076171875, + "loss": 1.1109, + "margin_dpo/margin_mean": 173.26296997070312, + "margin_dpo/margin_std": 267.32855224609375, + "step": 314 + }, + { + "KL/chosen_KL_mean": -272.3924865722656, + "KL/mean": -351.9873962402344, + "KL/rejected_KL_mean": -431.582275390625, + "KL/std": 185.17767333984375, + "epoch": 0.46255506607929514, + "fcm_dpo/beta": 0.0023773042485117912, + "fcm_dpo/delta": 0.02197282202541828, + "fcm_dpo/margin": 159.18983459472656, + "fcm_dpo/q_t": 0.4126739203929901, + "grad_norm": 25.598684310913086, + "learning_rate": 3.270101039870797e-07, + "logits/chosen": -0.27878469228744507, + "logits/rejected": -0.28102076053619385, + "logps/chosen": -321.46209716796875, + "logps/ref_chosen": -49.06958770751953, + "logps/ref_rejected": -85.68087768554688, + "logps/rejected": -517.26318359375, + "loss": 1.0996, + "margin_dpo/margin_mean": 159.18984985351562, + "margin_dpo/margin_std": 206.75869750976562, + "step": 315 + }, + { + "KL/chosen_KL_mean": -262.9434509277344, + "KL/mean": -372.22076416015625, + "KL/rejected_KL_mean": -481.4980773925781, + "KL/std": 218.68289184570312, + "epoch": 0.46402349486049926, + "fcm_dpo/beta": 0.0023532514460384846, + "fcm_dpo/delta": -0.11787203699350357, + "fcm_dpo/margin": 218.55462646484375, + "fcm_dpo/q_t": 0.38381052017211914, + "grad_norm": 26.231597900390625, + "learning_rate": 3.2578816852826086e-07, + "logits/chosen": -0.35521891713142395, + "logits/rejected": -0.35822099447250366, + "logps/chosen": -317.2041931152344, + "logps/ref_chosen": -54.26074981689453, + "logps/ref_rejected": -101.2814712524414, + "logps/rejected": -582.779541015625, + "loss": 1.0101, + "margin_dpo/margin_mean": 218.55462646484375, + "margin_dpo/margin_std": 240.36514282226562, + "step": 316 + }, + { + "KL/chosen_KL_mean": -265.1885681152344, + "KL/mean": -386.59716796875, + "KL/rejected_KL_mean": -508.0057373046875, + "KL/std": 205.4729461669922, + "epoch": 0.4654919236417034, + "fcm_dpo/beta": 0.0023275299463421106, + "fcm_dpo/delta": -0.1697898656129837, + "fcm_dpo/margin": 242.81719970703125, + "fcm_dpo/q_t": 0.37069547176361084, + "grad_norm": 29.441675186157227, + "learning_rate": 3.2456423598071783e-07, + "logits/chosen": -0.361335813999176, + "logits/rejected": -0.34446266293525696, + "logps/chosen": -321.28277587890625, + "logps/ref_chosen": -56.094207763671875, + "logps/ref_rejected": -100.69905090332031, + "logps/rejected": -608.704833984375, + "loss": 0.9674, + "margin_dpo/margin_mean": 242.81719970703125, + "margin_dpo/margin_std": 227.80020141601562, + "step": 317 + }, + { + "KL/chosen_KL_mean": -276.64666748046875, + "KL/mean": -368.34423828125, + "KL/rejected_KL_mean": -460.04180908203125, + "KL/std": 210.08200073242188, + "epoch": 0.4669603524229075, + "fcm_dpo/beta": 0.0023121268022805452, + "fcm_dpo/delta": -0.024627672508358955, + "fcm_dpo/margin": 183.39511108398438, + "fcm_dpo/q_t": 0.40299540758132935, + "grad_norm": 28.099123001098633, + "learning_rate": 3.233383385962115e-07, + "logits/chosen": -0.4194433093070984, + "logits/rejected": -0.38650479912757874, + "logps/chosen": -341.2923583984375, + "logps/ref_chosen": -64.64569854736328, + "logps/ref_rejected": -82.76425170898438, + "logps/rejected": -542.8060302734375, + "loss": 1.0716, + "margin_dpo/margin_mean": 183.3951416015625, + "margin_dpo/margin_std": 228.771728515625, + "step": 318 + }, + { + "KL/chosen_KL_mean": -248.31747436523438, + "KL/mean": -355.7877197265625, + "KL/rejected_KL_mean": -463.2579345703125, + "KL/std": 224.69723510742188, + "epoch": 0.4684287812041116, + "fcm_dpo/beta": 0.0022909443359822035, + "fcm_dpo/delta": -0.09473305940628052, + "fcm_dpo/margin": 214.94046020507812, + "fcm_dpo/q_t": 0.38672956824302673, + "grad_norm": 27.515636444091797, + "learning_rate": 3.2211050867827805e-07, + "logits/chosen": -0.365750253200531, + "logits/rejected": -0.37738245725631714, + "logps/chosen": -297.70123291015625, + "logps/ref_chosen": -49.383758544921875, + "logps/ref_rejected": -113.90650939941406, + "logps/rejected": -577.1644287109375, + "loss": 1.0225, + "margin_dpo/margin_mean": 214.94046020507812, + "margin_dpo/margin_std": 239.97752380371094, + "step": 319 + }, + { + "KL/chosen_KL_mean": -263.43206787109375, + "KL/mean": -381.2987365722656, + "KL/rejected_KL_mean": -499.1654052734375, + "KL/std": 234.49795532226562, + "epoch": 0.4698972099853157, + "fcm_dpo/beta": 0.0022584707476198673, + "fcm_dpo/delta": -0.13631628453731537, + "fcm_dpo/margin": 235.73333740234375, + "fcm_dpo/q_t": 0.3790479302406311, + "grad_norm": 35.816165924072266, + "learning_rate": 3.208807785813777e-07, + "logits/chosen": -0.35946518182754517, + "logits/rejected": -0.36113548278808594, + "logps/chosen": -322.93695068359375, + "logps/ref_chosen": -59.50489044189453, + "logps/ref_rejected": -97.66717529296875, + "logps/rejected": -596.83251953125, + "loss": 0.9952, + "margin_dpo/margin_mean": 235.73333740234375, + "margin_dpo/margin_std": 250.2997589111328, + "step": 320 + }, + { + "KL/chosen_KL_mean": -316.8851318359375, + "KL/mean": -419.2160339355469, + "KL/rejected_KL_mean": -521.546875, + "KL/std": 239.943115234375, + "epoch": 0.4713656387665198, + "fcm_dpo/beta": 0.002237518085166812, + "fcm_dpo/delta": -0.059808555990457535, + "fcm_dpo/margin": 204.66180419921875, + "fcm_dpo/q_t": 0.39591050148010254, + "grad_norm": 24.665813446044922, + "learning_rate": 3.1964918071004217e-07, + "logits/chosen": -0.3208162784576416, + "logits/rejected": -0.3067053556442261, + "logps/chosen": -378.433837890625, + "logps/ref_chosen": -61.548683166503906, + "logps/ref_rejected": -91.64103698730469, + "logps/rejected": -613.18798828125, + "loss": 1.0571, + "margin_dpo/margin_mean": 204.6617889404297, + "margin_dpo/margin_std": 253.84730529785156, + "step": 321 + }, + { + "KL/chosen_KL_mean": -280.55859375, + "KL/mean": -392.18109130859375, + "KL/rejected_KL_mean": -503.8035583496094, + "KL/std": 217.13014221191406, + "epoch": 0.47283406754772395, + "fcm_dpo/beta": 0.0022206236608326435, + "fcm_dpo/delta": -0.09844300150871277, + "fcm_dpo/margin": 223.24496459960938, + "fcm_dpo/q_t": 0.3849971294403076, + "grad_norm": 25.391277313232422, + "learning_rate": 3.184157475180207e-07, + "logits/chosen": -0.33665627241134644, + "logits/rejected": -0.33168351650238037, + "logps/chosen": -337.8486328125, + "logps/ref_chosen": -57.29003143310547, + "logps/ref_rejected": -95.74992370605469, + "logps/rejected": -599.553466796875, + "loss": 1.0101, + "margin_dpo/margin_mean": 223.24496459960938, + "margin_dpo/margin_std": 225.46527099609375, + "step": 322 + }, + { + "KL/chosen_KL_mean": -303.197509765625, + "KL/mean": -397.31976318359375, + "KL/rejected_KL_mean": -491.4420166015625, + "KL/std": 215.2032470703125, + "epoch": 0.47430249632892807, + "fcm_dpo/beta": 0.0022140834480524063, + "fcm_dpo/delta": -0.017185624688863754, + "fcm_dpo/margin": 188.2445068359375, + "fcm_dpo/q_t": 0.4033932685852051, + "grad_norm": 35.27677536010742, + "learning_rate": 3.171805115074251e-07, + "logits/chosen": -0.3552204370498657, + "logits/rejected": -0.3479047417640686, + "logps/chosen": -354.4314880371094, + "logps/ref_chosen": -51.23395919799805, + "logps/ref_rejected": -75.06192016601562, + "logps/rejected": -566.50390625, + "loss": 1.0792, + "margin_dpo/margin_mean": 188.2445068359375, + "margin_dpo/margin_std": 236.12969970703125, + "step": 323 + }, + { + "KL/chosen_KL_mean": -351.04327392578125, + "KL/mean": -444.7715759277344, + "KL/rejected_KL_mean": -538.4998779296875, + "KL/std": 236.59896850585938, + "epoch": 0.47577092511013214, + "fcm_dpo/beta": 0.0022205086424946785, + "fcm_dpo/delta": -0.01711501181125641, + "fcm_dpo/margin": 187.4566192626953, + "fcm_dpo/q_t": 0.4070439636707306, + "grad_norm": 41.267127990722656, + "learning_rate": 3.1594350522787295e-07, + "logits/chosen": -0.3706352710723877, + "logits/rejected": -0.35114365816116333, + "logps/chosen": -416.1784362792969, + "logps/ref_chosen": -65.13516998291016, + "logps/ref_rejected": -86.47750854492188, + "logps/rejected": -624.9774169921875, + "loss": 1.1032, + "margin_dpo/margin_mean": 187.4566192626953, + "margin_dpo/margin_std": 269.1782531738281, + "step": 324 + }, + { + "KL/chosen_KL_mean": -282.0893249511719, + "KL/mean": -358.64837646484375, + "KL/rejected_KL_mean": -435.2073974609375, + "KL/std": 205.12814331054688, + "epoch": 0.47723935389133626, + "fcm_dpo/beta": 0.0022224104031920433, + "fcm_dpo/delta": 0.06069522351026535, + "fcm_dpo/margin": 153.11805725097656, + "fcm_dpo/q_t": 0.4203560948371887, + "grad_norm": 36.374202728271484, + "learning_rate": 3.147047612756302e-07, + "logits/chosen": -0.4578849673271179, + "logits/rejected": -0.43039295077323914, + "logps/chosen": -338.304931640625, + "logps/ref_chosen": -56.215599060058594, + "logps/ref_rejected": -70.08592987060547, + "logps/rejected": -505.2933044433594, + "loss": 1.1272, + "margin_dpo/margin_mean": 153.11807250976562, + "margin_dpo/margin_std": 203.81394958496094, + "step": 325 + }, + { + "KL/chosen_KL_mean": -320.332763671875, + "KL/mean": -397.1983642578125, + "KL/rejected_KL_mean": -474.06396484375, + "KL/std": 202.8357696533203, + "epoch": 0.4787077826725404, + "fcm_dpo/beta": 0.0022304444573819637, + "fcm_dpo/delta": 0.0580889992415905, + "fcm_dpo/margin": 153.73118591308594, + "fcm_dpo/q_t": 0.41890084743499756, + "grad_norm": 45.461204528808594, + "learning_rate": 3.134643122927519e-07, + "logits/chosen": -0.44517433643341064, + "logits/rejected": -0.4201740622520447, + "logps/chosen": -393.0577087402344, + "logps/ref_chosen": -72.72496032714844, + "logps/ref_rejected": -79.8467788696289, + "logps/rejected": -553.9107666015625, + "loss": 1.1188, + "margin_dpo/margin_mean": 153.731201171875, + "margin_dpo/margin_std": 192.40139770507812, + "step": 326 + }, + { + "KL/chosen_KL_mean": -286.7748107910156, + "KL/mean": -405.4556884765625, + "KL/rejected_KL_mean": -524.1365356445312, + "KL/std": 219.92962646484375, + "epoch": 0.4801762114537445, + "fcm_dpo/beta": 0.0022127358242869377, + "fcm_dpo/delta": -0.1288842260837555, + "fcm_dpo/margin": 237.36172485351562, + "fcm_dpo/q_t": 0.3784708082675934, + "grad_norm": 38.16813659667969, + "learning_rate": 3.1222219096622264e-07, + "logits/chosen": -0.4000493884086609, + "logits/rejected": -0.38037049770355225, + "logps/chosen": -355.90924072265625, + "logps/ref_chosen": -69.13441467285156, + "logps/ref_rejected": -111.93377685546875, + "logps/rejected": -636.0703125, + "loss": 0.9955, + "margin_dpo/margin_mean": 237.36172485351562, + "margin_dpo/margin_std": 241.5782470703125, + "step": 327 + }, + { + "KL/chosen_KL_mean": -305.38720703125, + "KL/mean": -413.01141357421875, + "KL/rejected_KL_mean": -520.6356201171875, + "KL/std": 244.02554321289062, + "epoch": 0.48164464023494863, + "fcm_dpo/beta": 0.0021971408277750015, + "fcm_dpo/delta": -0.07468114793300629, + "fcm_dpo/margin": 215.24847412109375, + "fcm_dpo/q_t": 0.39337801933288574, + "grad_norm": 28.978845596313477, + "learning_rate": 3.1097843002709427e-07, + "logits/chosen": -0.369107723236084, + "logits/rejected": -0.367361843585968, + "logps/chosen": -365.07440185546875, + "logps/ref_chosen": -59.68719482421875, + "logps/ref_rejected": -90.85499572753906, + "logps/rejected": -611.4906616210938, + "loss": 1.0456, + "margin_dpo/margin_mean": 215.24847412109375, + "margin_dpo/margin_std": 267.3666687011719, + "step": 328 + }, + { + "KL/chosen_KL_mean": -340.37298583984375, + "KL/mean": -448.5474853515625, + "KL/rejected_KL_mean": -556.7220458984375, + "KL/std": 253.79428100585938, + "epoch": 0.4831130690161527, + "fcm_dpo/beta": 0.0021742871031165123, + "fcm_dpo/delta": -0.07264669984579086, + "fcm_dpo/margin": 216.34902954101562, + "fcm_dpo/q_t": 0.39304083585739136, + "grad_norm": 33.51917266845703, + "learning_rate": 3.0973306224962437e-07, + "logits/chosen": -0.37974807620048523, + "logits/rejected": -0.3639347553253174, + "logps/chosen": -405.61920166015625, + "logps/ref_chosen": -65.2461929321289, + "logps/ref_rejected": -100.69770812988281, + "logps/rejected": -657.4197387695312, + "loss": 1.0604, + "margin_dpo/margin_mean": 216.34902954101562, + "margin_dpo/margin_std": 278.5408630371094, + "step": 329 + }, + { + "KL/chosen_KL_mean": -288.67608642578125, + "KL/mean": -402.93505859375, + "KL/rejected_KL_mean": -517.1940307617188, + "KL/std": 241.49658203125, + "epoch": 0.4845814977973568, + "fcm_dpo/beta": 0.0021610576659440994, + "fcm_dpo/delta": -0.09621290862560272, + "fcm_dpo/margin": 228.5179443359375, + "fcm_dpo/q_t": 0.38732674717903137, + "grad_norm": 28.922462463378906, + "learning_rate": 3.084861204504122e-07, + "logits/chosen": -0.36659538745880127, + "logits/rejected": -0.3661782145500183, + "logps/chosen": -335.6744384765625, + "logps/ref_chosen": -46.998348236083984, + "logps/ref_rejected": -86.87684631347656, + "logps/rejected": -604.0708618164062, + "loss": 1.0234, + "margin_dpo/margin_mean": 228.5179443359375, + "margin_dpo/margin_std": 258.7063903808594, + "step": 330 + }, + { + "KL/chosen_KL_mean": -310.08087158203125, + "KL/mean": -423.1429443359375, + "KL/rejected_KL_mean": -536.2050170898438, + "KL/std": 198.00454711914062, + "epoch": 0.48604992657856094, + "fcm_dpo/beta": 0.002141648903489113, + "fcm_dpo/delta": -0.08637881278991699, + "fcm_dpo/margin": 226.1241455078125, + "fcm_dpo/q_t": 0.3862583339214325, + "grad_norm": 28.123804092407227, + "learning_rate": 3.072376374875335e-07, + "logits/chosen": -0.4031466245651245, + "logits/rejected": -0.395566463470459, + "logps/chosen": -360.6051025390625, + "logps/ref_chosen": -50.52424621582031, + "logps/ref_rejected": -89.01544189453125, + "logps/rejected": -625.220458984375, + "loss": 0.9994, + "margin_dpo/margin_mean": 226.1241455078125, + "margin_dpo/margin_std": 195.76531982421875, + "step": 331 + }, + { + "KL/chosen_KL_mean": -316.3252258300781, + "KL/mean": -394.60198974609375, + "KL/rejected_KL_mean": -472.87872314453125, + "KL/std": 199.60614013671875, + "epoch": 0.48751835535976507, + "fcm_dpo/beta": 0.002148597501218319, + "fcm_dpo/delta": 0.06467652320861816, + "fcm_dpo/margin": 156.5535125732422, + "fcm_dpo/q_t": 0.42228198051452637, + "grad_norm": 25.59779930114746, + "learning_rate": 3.059876462596758e-07, + "logits/chosen": -0.3974687159061432, + "logits/rejected": -0.37797918915748596, + "logps/chosen": -365.5054931640625, + "logps/ref_chosen": -49.18028259277344, + "logps/ref_rejected": -76.48515319824219, + "logps/rejected": -549.3638916015625, + "loss": 1.1305, + "margin_dpo/margin_mean": 156.55352783203125, + "margin_dpo/margin_std": 220.91140747070312, + "step": 332 + }, + { + "KL/chosen_KL_mean": -325.0455627441406, + "KL/mean": -431.7874450683594, + "KL/rejected_KL_mean": -538.529296875, + "KL/std": 234.52767944335938, + "epoch": 0.4889867841409692, + "fcm_dpo/beta": 0.0021342344116419554, + "fcm_dpo/delta": -0.057633012533187866, + "fcm_dpo/margin": 213.48370361328125, + "fcm_dpo/q_t": 0.3971046805381775, + "grad_norm": 30.5307674407959, + "learning_rate": 3.0473617970527015e-07, + "logits/chosen": -0.4062424302101135, + "logits/rejected": -0.3987647593021393, + "logps/chosen": -388.80133056640625, + "logps/ref_chosen": -63.75574493408203, + "logps/ref_rejected": -95.04411315917969, + "logps/rejected": -633.5733642578125, + "loss": 1.0699, + "margin_dpo/margin_mean": 213.48370361328125, + "margin_dpo/margin_std": 282.26739501953125, + "step": 333 + }, + { + "KL/chosen_KL_mean": -314.6381530761719, + "KL/mean": -410.3465270996094, + "KL/rejected_KL_mean": -506.054931640625, + "KL/std": 256.6167907714844, + "epoch": 0.49045521292217326, + "fcm_dpo/beta": 0.0021365054417401552, + "fcm_dpo/delta": -0.00915764831006527, + "fcm_dpo/margin": 191.416748046875, + "fcm_dpo/q_t": 0.4075648784637451, + "grad_norm": 27.895448684692383, + "learning_rate": 3.034832708016243e-07, + "logits/chosen": -0.4134640693664551, + "logits/rejected": -0.40898776054382324, + "logps/chosen": -381.617919921875, + "logps/ref_chosen": -66.97975158691406, + "logps/ref_rejected": -95.31692504882812, + "logps/rejected": -601.371826171875, + "loss": 1.1071, + "margin_dpo/margin_mean": 191.416748046875, + "margin_dpo/margin_std": 287.8053283691406, + "step": 334 + }, + { + "KL/chosen_KL_mean": -340.8347473144531, + "KL/mean": -420.40362548828125, + "KL/rejected_KL_mean": -499.97247314453125, + "KL/std": 239.92379760742188, + "epoch": 0.4919236417033774, + "fcm_dpo/beta": 0.00215163454413414, + "fcm_dpo/delta": 0.058172550052404404, + "fcm_dpo/margin": 159.1376953125, + "fcm_dpo/q_t": 0.4224546253681183, + "grad_norm": 31.30469512939453, + "learning_rate": 3.022289525640531e-07, + "logits/chosen": -0.47105371952056885, + "logits/rejected": -0.44997382164001465, + "logps/chosen": -403.3772277832031, + "logps/ref_chosen": -62.54248046875, + "logps/ref_rejected": -87.61770629882812, + "logps/rejected": -587.5902099609375, + "loss": 1.147, + "margin_dpo/margin_mean": 159.1376953125, + "margin_dpo/margin_std": 256.5185241699219, + "step": 335 + }, + { + "KL/chosen_KL_mean": -337.67694091796875, + "KL/mean": -456.9693603515625, + "KL/rejected_KL_mean": -576.26171875, + "KL/std": 266.134033203125, + "epoch": 0.4933920704845815, + "fcm_dpo/beta": 0.0021344278939068317, + "fcm_dpo/delta": -0.1120191439986229, + "fcm_dpo/margin": 238.58480834960938, + "fcm_dpo/q_t": 0.3868195414543152, + "grad_norm": 38.22102737426758, + "learning_rate": 3.009732580450086e-07, + "logits/chosen": -0.4278082847595215, + "logits/rejected": -0.4280624985694885, + "logps/chosen": -392.20806884765625, + "logps/ref_chosen": -54.53115463256836, + "logps/ref_rejected": -104.40424346923828, + "logps/rejected": -680.666015625, + "loss": 1.0426, + "margin_dpo/margin_mean": 238.58480834960938, + "margin_dpo/margin_std": 310.658447265625, + "step": 336 + }, + { + "KL/chosen_KL_mean": -302.09124755859375, + "KL/mean": -415.0406494140625, + "KL/rejected_KL_mean": -527.9901123046875, + "KL/std": 218.13992309570312, + "epoch": 0.4948604992657856, + "fcm_dpo/beta": 0.0021110186353325844, + "fcm_dpo/delta": -0.0788479745388031, + "fcm_dpo/margin": 225.89886474609375, + "fcm_dpo/q_t": 0.39037024974823, + "grad_norm": 39.11850357055664, + "learning_rate": 2.9971622033320914e-07, + "logits/chosen": -0.4359098970890045, + "logits/rejected": -0.4225747585296631, + "logps/chosen": -367.21990966796875, + "logps/ref_chosen": -65.12869262695312, + "logps/ref_rejected": -101.72701263427734, + "logps/rejected": -629.7171020507812, + "loss": 1.0344, + "margin_dpo/margin_mean": 225.89886474609375, + "margin_dpo/margin_std": 259.23974609375, + "step": 337 + }, + { + "KL/chosen_KL_mean": -288.96722412109375, + "KL/mean": -404.56036376953125, + "KL/rejected_KL_mean": -520.1534423828125, + "KL/std": 224.17669677734375, + "epoch": 0.49632892804698975, + "fcm_dpo/beta": 0.002094029448926449, + "fcm_dpo/delta": -0.08624735474586487, + "fcm_dpo/margin": 231.18624877929688, + "fcm_dpo/q_t": 0.3876437246799469, + "grad_norm": 39.132667541503906, + "learning_rate": 2.984578725527675e-07, + "logits/chosen": -0.4282386302947998, + "logits/rejected": -0.42397069931030273, + "logps/chosen": -347.3899230957031, + "logps/ref_chosen": -58.422706604003906, + "logps/ref_rejected": -89.06854248046875, + "logps/rejected": -609.2219848632812, + "loss": 1.016, + "margin_dpo/margin_mean": 231.18624877929688, + "margin_dpo/margin_std": 235.46348571777344, + "step": 338 + }, + { + "KL/chosen_KL_mean": -302.4131774902344, + "KL/mean": -405.36090087890625, + "KL/rejected_KL_mean": -508.3086242675781, + "KL/std": 225.501708984375, + "epoch": 0.4977973568281938, + "fcm_dpo/beta": 0.00209475914016366, + "fcm_dpo/delta": -0.032657869160175323, + "fcm_dpo/margin": 205.89544677734375, + "fcm_dpo/q_t": 0.4000750184059143, + "grad_norm": 39.0589485168457, + "learning_rate": 2.9719824786231796e-07, + "logits/chosen": -0.4808998703956604, + "logits/rejected": -0.4646437168121338, + "logps/chosen": -362.40850830078125, + "logps/ref_chosen": -59.99531555175781, + "logps/ref_rejected": -103.9109115600586, + "logps/rejected": -612.219482421875, + "loss": 1.0614, + "margin_dpo/margin_mean": 205.89544677734375, + "margin_dpo/margin_std": 227.13043212890625, + "step": 339 + }, + { + "KL/chosen_KL_mean": -306.40838623046875, + "KL/mean": -403.55328369140625, + "KL/rejected_KL_mean": -500.6982116699219, + "KL/std": 214.56753540039062, + "epoch": 0.49926578560939794, + "fcm_dpo/beta": 0.0020795511081814766, + "fcm_dpo/delta": -0.004235394299030304, + "fcm_dpo/margin": 194.28976440429688, + "fcm_dpo/q_t": 0.40789303183555603, + "grad_norm": 32.46416091918945, + "learning_rate": 2.959373794541426e-07, + "logits/chosen": -0.42205601930618286, + "logits/rejected": -0.4012700319290161, + "logps/chosen": -359.2386474609375, + "logps/ref_chosen": -52.83022689819336, + "logps/ref_rejected": -73.10723114013672, + "logps/rejected": -573.805419921875, + "loss": 1.0937, + "margin_dpo/margin_mean": 194.28977966308594, + "margin_dpo/margin_std": 265.2552185058594, + "step": 340 + }, + { + "KL/chosen_KL_mean": -299.121826171875, + "KL/mean": -406.19964599609375, + "KL/rejected_KL_mean": -513.2774658203125, + "KL/std": 231.76580810546875, + "epoch": 0.5007342143906021, + "fcm_dpo/beta": 0.0020710937678813934, + "fcm_dpo/delta": -0.044832952320575714, + "fcm_dpo/margin": 214.15560913085938, + "fcm_dpo/q_t": 0.398419588804245, + "grad_norm": 32.579837799072266, + "learning_rate": 2.946753005532965e-07, + "logits/chosen": -0.42187175154685974, + "logits/rejected": -0.4231615364551544, + "logps/chosen": -347.0216369628906, + "logps/ref_chosen": -47.899803161621094, + "logps/ref_rejected": -101.80987548828125, + "logps/rejected": -615.0873413085938, + "loss": 1.0562, + "margin_dpo/margin_mean": 214.15560913085938, + "margin_dpo/margin_std": 249.23519897460938, + "step": 341 + }, + { + "KL/chosen_KL_mean": -285.2144470214844, + "KL/mean": -383.55755615234375, + "KL/rejected_KL_mean": -481.900634765625, + "KL/std": 211.0296173095703, + "epoch": 0.5022026431718062, + "fcm_dpo/beta": 0.0020756039302796125, + "fcm_dpo/delta": -0.008514024317264557, + "fcm_dpo/margin": 196.68621826171875, + "fcm_dpo/q_t": 0.40535086393356323, + "grad_norm": 29.37921905517578, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": -0.41295289993286133, + "logits/rejected": -0.3835510313510895, + "logps/chosen": -357.2110900878906, + "logps/ref_chosen": -71.99664306640625, + "logps/ref_rejected": -92.58959197998047, + "logps/rejected": -574.490234375, + "loss": 1.0793, + "margin_dpo/margin_mean": 196.68621826171875, + "margin_dpo/margin_std": 243.93203735351562, + "step": 342 + }, + { + "KL/chosen_KL_mean": -275.50653076171875, + "KL/mean": -393.09033203125, + "KL/rejected_KL_mean": -510.6741027832031, + "KL/std": 219.71334838867188, + "epoch": 0.5036710719530103, + "fcm_dpo/beta": 0.0020575113594532013, + "fcm_dpo/delta": -0.08598469197750092, + "fcm_dpo/margin": 235.1676025390625, + "fcm_dpo/q_t": 0.38554584980010986, + "grad_norm": 25.824504852294922, + "learning_rate": 2.9214764433242476e-07, + "logits/chosen": -0.4432937502861023, + "logits/rejected": -0.4439564347267151, + "logps/chosen": -329.91217041015625, + "logps/ref_chosen": -54.405616760253906, + "logps/ref_rejected": -111.04142761230469, + "logps/rejected": -621.715576171875, + "loss": 0.9987, + "margin_dpo/margin_mean": 235.16761779785156, + "margin_dpo/margin_std": 193.387451171875, + "step": 343 + }, + { + "KL/chosen_KL_mean": -278.17584228515625, + "KL/mean": -382.2356262207031, + "KL/rejected_KL_mean": -486.29541015625, + "KL/std": 239.05442810058594, + "epoch": 0.5051395007342144, + "fcm_dpo/beta": 0.00205832626670599, + "fcm_dpo/delta": -0.029630951583385468, + "fcm_dpo/margin": 208.11956787109375, + "fcm_dpo/q_t": 0.40320706367492676, + "grad_norm": 28.157310485839844, + "learning_rate": 2.9088213361849126e-07, + "logits/chosen": -0.421103835105896, + "logits/rejected": -0.42124348878860474, + "logps/chosen": -332.1405029296875, + "logps/ref_chosen": -53.96466827392578, + "logps/ref_rejected": -90.62336730957031, + "logps/rejected": -576.9188232421875, + "loss": 1.066, + "margin_dpo/margin_mean": 208.11956787109375, + "margin_dpo/margin_std": 241.80572509765625, + "step": 344 + }, + { + "KL/chosen_KL_mean": -327.5186767578125, + "KL/mean": -443.9566650390625, + "KL/rejected_KL_mean": -560.3946533203125, + "KL/std": 230.4110107421875, + "epoch": 0.5066079295154186, + "fcm_dpo/beta": 0.0020382110960781574, + "fcm_dpo/delta": -0.0764290988445282, + "fcm_dpo/margin": 232.87599182128906, + "fcm_dpo/q_t": 0.3901875615119934, + "grad_norm": 24.22003173828125, + "learning_rate": 2.896155456223163e-07, + "logits/chosen": -0.43897050619125366, + "logits/rejected": -0.4347718358039856, + "logps/chosen": -389.2043762207031, + "logps/ref_chosen": -61.685699462890625, + "logps/ref_rejected": -99.49041748046875, + "logps/rejected": -659.8851318359375, + "loss": 1.0293, + "margin_dpo/margin_mean": 232.87600708007812, + "margin_dpo/margin_std": 254.49136352539062, + "step": 345 + }, + { + "KL/chosen_KL_mean": -339.9031677246094, + "KL/mean": -448.6493835449219, + "KL/rejected_KL_mean": -557.3956298828125, + "KL/std": 233.10879516601562, + "epoch": 0.5080763582966226, + "fcm_dpo/beta": 0.0020257416181266308, + "fcm_dpo/delta": -0.04150301218032837, + "fcm_dpo/margin": 217.49244689941406, + "fcm_dpo/q_t": 0.3968901038169861, + "grad_norm": 27.246360778808594, + "learning_rate": 2.883479137196714e-07, + "logits/chosen": -0.3788645267486572, + "logits/rejected": -0.3643970489501953, + "logps/chosen": -395.159423828125, + "logps/ref_chosen": -55.256263732910156, + "logps/ref_rejected": -77.41532135009766, + "logps/rejected": -634.8109130859375, + "loss": 1.0528, + "margin_dpo/margin_mean": 217.49244689941406, + "margin_dpo/margin_std": 247.9464111328125, + "step": 346 + }, + { + "KL/chosen_KL_mean": -344.7647705078125, + "KL/mean": -448.3440246582031, + "KL/rejected_KL_mean": -551.9232177734375, + "KL/std": 247.40902709960938, + "epoch": 0.5095447870778267, + "fcm_dpo/beta": 0.002020738087594509, + "fcm_dpo/delta": -0.019013745710253716, + "fcm_dpo/margin": 207.15847778320312, + "fcm_dpo/q_t": 0.4043384790420532, + "grad_norm": 29.887693405151367, + "learning_rate": 2.8707927131383614e-07, + "logits/chosen": -0.3656623959541321, + "logits/rejected": -0.35582268238067627, + "logps/chosen": -402.3310241699219, + "logps/ref_chosen": -57.56623840332031, + "logps/ref_rejected": -92.35509490966797, + "logps/rejected": -644.2783203125, + "loss": 1.0832, + "margin_dpo/margin_mean": 207.15847778320312, + "margin_dpo/margin_std": 276.2279052734375, + "step": 347 + }, + { + "KL/chosen_KL_mean": -311.39959716796875, + "KL/mean": -401.85382080078125, + "KL/rejected_KL_mean": -492.3080749511719, + "KL/std": 215.942626953125, + "epoch": 0.5110132158590308, + "fcm_dpo/beta": 0.002018729457631707, + "fcm_dpo/delta": 0.03512571379542351, + "fcm_dpo/margin": 180.908447265625, + "fcm_dpo/q_t": 0.4162946939468384, + "grad_norm": 29.292268753051758, + "learning_rate": 2.858096518347179e-07, + "logits/chosen": -0.3954634666442871, + "logits/rejected": -0.3925415873527527, + "logps/chosen": -367.7173156738281, + "logps/ref_chosen": -56.31770324707031, + "logps/ref_rejected": -89.13836669921875, + "logps/rejected": -581.4464111328125, + "loss": 1.1176, + "margin_dpo/margin_mean": 180.90846252441406, + "margin_dpo/margin_std": 251.56272888183594, + "step": 348 + }, + { + "KL/chosen_KL_mean": -302.69281005859375, + "KL/mean": -409.18768310546875, + "KL/rejected_KL_mean": -515.6825561523438, + "KL/std": 244.12184143066406, + "epoch": 0.5124816446402349, + "fcm_dpo/beta": 0.002026638016104698, + "fcm_dpo/delta": -0.03247387707233429, + "fcm_dpo/margin": 212.98977661132812, + "fcm_dpo/q_t": 0.40324437618255615, + "grad_norm": 24.500629425048828, + "learning_rate": 2.845390887379706e-07, + "logits/chosen": -0.37826380133628845, + "logits/rejected": -0.37416696548461914, + "logps/chosen": -360.71832275390625, + "logps/ref_chosen": -58.025516510009766, + "logps/ref_rejected": -97.50515747070312, + "logps/rejected": -613.187744140625, + "loss": 1.0867, + "margin_dpo/margin_mean": 212.98977661132812, + "margin_dpo/margin_std": 299.3753662109375, + "step": 349 + }, + { + "KL/chosen_KL_mean": -322.9959716796875, + "KL/mean": -425.35693359375, + "KL/rejected_KL_mean": -527.7179565429688, + "KL/std": 237.5716552734375, + "epoch": 0.5139500734214391, + "fcm_dpo/beta": 0.002013745252043009, + "fcm_dpo/delta": -0.012782763689756393, + "fcm_dpo/margin": 204.7219696044922, + "fcm_dpo/q_t": 0.40567225217819214, + "grad_norm": 32.45606994628906, + "learning_rate": 2.8326761550411346e-07, + "logits/chosen": -0.4236910343170166, + "logits/rejected": -0.4241228699684143, + "logps/chosen": -387.32647705078125, + "logps/ref_chosen": -64.33049011230469, + "logps/ref_rejected": -89.87164306640625, + "logps/rejected": -617.589599609375, + "loss": 1.1016, + "margin_dpo/margin_mean": 204.72198486328125, + "margin_dpo/margin_std": 298.5418701171875, + "step": 350 + }, + { + "KL/chosen_KL_mean": -301.8697509765625, + "KL/mean": -426.64111328125, + "KL/rejected_KL_mean": -551.4124755859375, + "KL/std": 277.66021728515625, + "epoch": 0.5154185022026432, + "fcm_dpo/beta": 0.0019994243048131466, + "fcm_dpo/delta": -0.10174670815467834, + "fcm_dpo/margin": 249.542724609375, + "fcm_dpo/q_t": 0.39052557945251465, + "grad_norm": 26.109012603759766, + "learning_rate": 2.819952656376487e-07, + "logits/chosen": -0.4383613169193268, + "logits/rejected": -0.434881329536438, + "logps/chosen": -362.5418701171875, + "logps/ref_chosen": -60.6721305847168, + "logps/ref_rejected": -101.5654296875, + "logps/rejected": -652.9779052734375, + "loss": 1.047, + "margin_dpo/margin_mean": 249.542724609375, + "margin_dpo/margin_std": 331.46405029296875, + "step": 351 + }, + { + "KL/chosen_KL_mean": -353.83587646484375, + "KL/mean": -431.2773132324219, + "KL/rejected_KL_mean": -508.7187194824219, + "KL/std": 258.7840576171875, + "epoch": 0.5168869309838473, + "fcm_dpo/beta": 0.002007380360737443, + "fcm_dpo/delta": 0.09052670747041702, + "fcm_dpo/margin": 154.8828887939453, + "fcm_dpo/q_t": 0.4296043813228607, + "grad_norm": 35.03353500366211, + "learning_rate": 2.8072207266617854e-07, + "logits/chosen": -0.4138913154602051, + "logits/rejected": -0.381538987159729, + "logps/chosen": -424.7793273925781, + "logps/ref_chosen": -70.9434585571289, + "logps/ref_rejected": -76.6419677734375, + "logps/rejected": -585.3607177734375, + "loss": 1.1807, + "margin_dpo/margin_mean": 154.8828887939453, + "margin_dpo/margin_std": 290.50494384765625, + "step": 352 + }, + { + "KL/chosen_KL_mean": -340.01251220703125, + "KL/mean": -443.328857421875, + "KL/rejected_KL_mean": -546.6452026367188, + "KL/std": 257.87725830078125, + "epoch": 0.5183553597650514, + "fcm_dpo/beta": 0.002007407834753394, + "fcm_dpo/delta": -0.015510857105255127, + "fcm_dpo/margin": 206.63265991210938, + "fcm_dpo/q_t": 0.40632784366607666, + "grad_norm": 29.06142234802246, + "learning_rate": 2.794480701395219e-07, + "logits/chosen": -0.46683937311172485, + "logits/rejected": -0.4540403485298157, + "logps/chosen": -398.4078674316406, + "logps/ref_chosen": -58.39533996582031, + "logps/ref_rejected": -80.33553314208984, + "logps/rejected": -626.980712890625, + "loss": 1.0954, + "margin_dpo/margin_mean": 206.63265991210938, + "margin_dpo/margin_std": 289.2551574707031, + "step": 353 + }, + { + "KL/chosen_KL_mean": -282.2817687988281, + "KL/mean": -397.5120544433594, + "KL/rejected_KL_mean": -512.7423095703125, + "KL/std": 236.1700439453125, + "epoch": 0.5198237885462555, + "fcm_dpo/beta": 0.0020015356130898, + "fcm_dpo/delta": -0.062798872590065, + "fcm_dpo/margin": 230.4605712890625, + "fcm_dpo/q_t": 0.39260250329971313, + "grad_norm": 30.49873161315918, + "learning_rate": 2.781732916288303e-07, + "logits/chosen": -0.3843461275100708, + "logits/rejected": -0.36748528480529785, + "logps/chosen": -342.08477783203125, + "logps/ref_chosen": -59.80299377441406, + "logps/ref_rejected": -88.75750732421875, + "logps/rejected": -601.4998779296875, + "loss": 1.0285, + "margin_dpo/margin_mean": 230.4605712890625, + "margin_dpo/margin_std": 233.89678955078125, + "step": 354 + }, + { + "KL/chosen_KL_mean": -279.9733581542969, + "KL/mean": -393.3092041015625, + "KL/rejected_KL_mean": -506.6451110839844, + "KL/std": 233.89459228515625, + "epoch": 0.5212922173274597, + "fcm_dpo/beta": 0.001989755779504776, + "fcm_dpo/delta": -0.05232168734073639, + "fcm_dpo/margin": 226.67178344726562, + "fcm_dpo/q_t": 0.39414530992507935, + "grad_norm": 42.037132263183594, + "learning_rate": 2.7689777072570284e-07, + "logits/chosen": -0.4780992865562439, + "logits/rejected": -0.45962148904800415, + "logps/chosen": -334.10186767578125, + "logps/ref_chosen": -54.12849807739258, + "logps/ref_rejected": -82.40606689453125, + "logps/rejected": -589.0511474609375, + "loss": 1.0331, + "margin_dpo/margin_mean": 226.67176818847656, + "margin_dpo/margin_std": 225.0481414794922, + "step": 355 + }, + { + "KL/chosen_KL_mean": -339.6640319824219, + "KL/mean": -402.67962646484375, + "KL/rejected_KL_mean": -465.6952819824219, + "KL/std": 238.7618865966797, + "epoch": 0.5227606461086637, + "fcm_dpo/beta": 0.001991572557017207, + "fcm_dpo/delta": 0.020430708304047585, + "fcm_dpo/margin": 126.03125, + "fcm_dpo/q_t": 0.44326043128967285, + "grad_norm": 35.44585418701172, + "learning_rate": 2.7562154104130176e-07, + "logits/chosen": -0.43858081102371216, + "logits/rejected": -0.4161129593849182, + "logps/chosen": -404.33782958984375, + "logps/ref_chosen": -64.6738052368164, + "logps/ref_rejected": -75.89926147460938, + "logps/rejected": -541.5945434570312, + "loss": 1.2348, + "margin_dpo/margin_mean": 126.03125762939453, + "margin_dpo/margin_std": 294.35003662109375, + "step": 356 + }, + { + "KL/chosen_KL_mean": -307.77178955078125, + "KL/mean": -404.3070068359375, + "KL/rejected_KL_mean": -500.8422546386719, + "KL/std": 227.6900634765625, + "epoch": 0.5242290748898678, + "fcm_dpo/beta": 0.001992112258449197, + "fcm_dpo/delta": 0.015565093606710434, + "fcm_dpo/margin": 193.07044982910156, + "fcm_dpo/q_t": 0.40995845198631287, + "grad_norm": 27.028295516967773, + "learning_rate": 2.7434463620546594e-07, + "logits/chosen": -0.4511488080024719, + "logits/rejected": -0.43502819538116455, + "logps/chosen": -360.49761962890625, + "logps/ref_chosen": -52.725799560546875, + "logps/ref_rejected": -86.84115600585938, + "logps/rejected": -587.6834106445312, + "loss": 1.0897, + "margin_dpo/margin_mean": 193.07044982910156, + "margin_dpo/margin_std": 232.12307739257812, + "step": 357 + }, + { + "KL/chosen_KL_mean": -285.88421630859375, + "KL/mean": -376.27392578125, + "KL/rejected_KL_mean": -466.66363525390625, + "KL/std": 219.6377716064453, + "epoch": 0.5256975036710719, + "fcm_dpo/beta": 0.0020017188508063555, + "fcm_dpo/delta": 0.0388420931994915, + "fcm_dpo/margin": 180.77944946289062, + "fcm_dpo/q_t": 0.4163302183151245, + "grad_norm": 24.81241798400879, + "learning_rate": 2.730670898658255e-07, + "logits/chosen": -0.4996035397052765, + "logits/rejected": -0.4798714816570282, + "logps/chosen": -349.08966064453125, + "logps/ref_chosen": -63.20543670654297, + "logps/ref_rejected": -88.373291015625, + "logps/rejected": -555.0369873046875, + "loss": 1.1092, + "margin_dpo/margin_mean": 180.77944946289062, + "margin_dpo/margin_std": 236.979248046875, + "step": 358 + }, + { + "KL/chosen_KL_mean": -329.12579345703125, + "KL/mean": -436.353271484375, + "KL/rejected_KL_mean": -543.5807495117188, + "KL/std": 229.46609497070312, + "epoch": 0.527165932452276, + "fcm_dpo/beta": 0.0019993209280073643, + "fcm_dpo/delta": -0.02941155806183815, + "fcm_dpo/margin": 214.4549560546875, + "fcm_dpo/q_t": 0.4025813341140747, + "grad_norm": 51.26544952392578, + "learning_rate": 2.717889356869146e-07, + "logits/chosen": -0.4144907593727112, + "logits/rejected": -0.3954973518848419, + "logps/chosen": -385.4959716796875, + "logps/ref_chosen": -56.370216369628906, + "logps/ref_rejected": -82.17375183105469, + "logps/rejected": -625.7545166015625, + "loss": 1.076, + "margin_dpo/margin_mean": 214.4549560546875, + "margin_dpo/margin_std": 278.1531982421875, + "step": 359 + }, + { + "KL/chosen_KL_mean": -320.970703125, + "KL/mean": -405.2179870605469, + "KL/rejected_KL_mean": -489.4652404785156, + "KL/std": 195.44993591308594, + "epoch": 0.5286343612334802, + "fcm_dpo/beta": 0.002009341726079583, + "fcm_dpo/delta": 0.06243222951889038, + "fcm_dpo/margin": 168.49453735351562, + "fcm_dpo/q_t": 0.4197266101837158, + "grad_norm": 39.978214263916016, + "learning_rate": 2.7051020734928443e-07, + "logits/chosen": -0.41467490792274475, + "logits/rejected": -0.39842575788497925, + "logps/chosen": -372.43109130859375, + "logps/ref_chosen": -51.460384368896484, + "logps/ref_rejected": -69.83892059326172, + "logps/rejected": -559.30419921875, + "loss": 1.1129, + "margin_dpo/margin_mean": 168.49453735351562, + "margin_dpo/margin_std": 194.20201110839844, + "step": 360 + }, + { + "KL/chosen_KL_mean": -342.26776123046875, + "KL/mean": -424.4127197265625, + "KL/rejected_KL_mean": -506.5577087402344, + "KL/std": 229.03176879882812, + "epoch": 0.5301027900146843, + "fcm_dpo/beta": 0.0020234398543834686, + "fcm_dpo/delta": 0.06860539317131042, + "fcm_dpo/margin": 164.2899627685547, + "fcm_dpo/q_t": 0.42270296812057495, + "grad_norm": 31.659528732299805, + "learning_rate": 2.6923093854861593e-07, + "logits/chosen": -0.46579843759536743, + "logits/rejected": -0.45978784561157227, + "logps/chosen": -396.13726806640625, + "logps/ref_chosen": -53.86951446533203, + "logps/ref_rejected": -90.7692642211914, + "logps/rejected": -597.3270263671875, + "loss": 1.1397, + "margin_dpo/margin_mean": 164.2899627685547, + "margin_dpo/margin_std": 245.25772094726562, + "step": 361 + }, + { + "KL/chosen_KL_mean": -297.3023376464844, + "KL/mean": -432.37384033203125, + "KL/rejected_KL_mean": -567.4454345703125, + "KL/std": 242.36074829101562, + "epoch": 0.5315712187958884, + "fcm_dpo/beta": 0.0019996415358036757, + "fcm_dpo/delta": -0.14447136223316193, + "fcm_dpo/margin": 270.14306640625, + "fcm_dpo/q_t": 0.3760761320590973, + "grad_norm": 30.424165725708008, + "learning_rate": 2.679511629948319e-07, + "logits/chosen": -0.47015565633773804, + "logits/rejected": -0.47379571199417114, + "logps/chosen": -355.94140625, + "logps/ref_chosen": -58.639060974121094, + "logps/ref_rejected": -105.58195495605469, + "logps/rejected": -673.02734375, + "loss": 0.9747, + "margin_dpo/margin_mean": 270.14306640625, + "margin_dpo/margin_std": 252.4774169921875, + "step": 362 + }, + { + "KL/chosen_KL_mean": -270.51788330078125, + "KL/mean": -407.674560546875, + "KL/rejected_KL_mean": -544.8311767578125, + "KL/std": 241.01730346679688, + "epoch": 0.5330396475770925, + "fcm_dpo/beta": 0.0019766553305089474, + "fcm_dpo/delta": -0.1460692584514618, + "fcm_dpo/margin": 274.31329345703125, + "fcm_dpo/q_t": 0.3755940794944763, + "grad_norm": 27.271324157714844, + "learning_rate": 2.6667091441120816e-07, + "logits/chosen": -0.4525992274284363, + "logits/rejected": -0.43701645731925964, + "logps/chosen": -315.0762939453125, + "logps/ref_chosen": -44.558380126953125, + "logps/ref_rejected": -74.69496154785156, + "logps/rejected": -619.526123046875, + "loss": 0.9773, + "margin_dpo/margin_mean": 274.31329345703125, + "margin_dpo/margin_std": 263.22528076171875, + "step": 363 + }, + { + "KL/chosen_KL_mean": -289.7828369140625, + "KL/mean": -389.1034240722656, + "KL/rejected_KL_mean": -488.4240417480469, + "KL/std": 230.4542236328125, + "epoch": 0.5345080763582967, + "fcm_dpo/beta": 0.0019719074480235577, + "fcm_dpo/delta": 0.008304569870233536, + "fcm_dpo/margin": 198.64120483398438, + "fcm_dpo/q_t": 0.40914398431777954, + "grad_norm": 25.84320640563965, + "learning_rate": 2.6539022653348575e-07, + "logits/chosen": -0.4640185236930847, + "logits/rejected": -0.46951600909233093, + "logps/chosen": -338.67742919921875, + "logps/ref_chosen": -48.894622802734375, + "logps/ref_rejected": -91.395751953125, + "logps/rejected": -579.81982421875, + "loss": 1.0964, + "margin_dpo/margin_mean": 198.64120483398438, + "margin_dpo/margin_std": 262.0460205078125, + "step": 364 + }, + { + "KL/chosen_KL_mean": -278.07470703125, + "KL/mean": -384.00201416015625, + "KL/rejected_KL_mean": -489.9293518066406, + "KL/std": 242.47592163085938, + "epoch": 0.5359765051395007, + "fcm_dpo/beta": 0.0019668107852339745, + "fcm_dpo/delta": -0.01703200675547123, + "fcm_dpo/margin": 211.85464477539062, + "fcm_dpo/q_t": 0.40431296825408936, + "grad_norm": 24.915437698364258, + "learning_rate": 2.641091331089811e-07, + "logits/chosen": -0.4698528051376343, + "logits/rejected": -0.4782125949859619, + "logps/chosen": -329.56744384765625, + "logps/ref_chosen": -51.49274444580078, + "logps/ref_rejected": -92.70166778564453, + "logps/rejected": -582.6309814453125, + "loss": 1.0661, + "margin_dpo/margin_mean": 211.8546600341797, + "margin_dpo/margin_std": 245.96029663085938, + "step": 365 + }, + { + "KL/chosen_KL_mean": -254.35284423828125, + "KL/mean": -363.2435302734375, + "KL/rejected_KL_mean": -472.13421630859375, + "KL/std": 230.414306640625, + "epoch": 0.5374449339207048, + "fcm_dpo/beta": 0.001956676598638296, + "fcm_dpo/delta": -0.02698526531457901, + "fcm_dpo/margin": 217.7813720703125, + "fcm_dpo/q_t": 0.40227261185646057, + "grad_norm": 23.5860652923584, + "learning_rate": 2.6282766789569736e-07, + "logits/chosen": -0.44011300802230835, + "logits/rejected": -0.4551384449005127, + "logps/chosen": -299.07342529296875, + "logps/ref_chosen": -44.7205696105957, + "logps/ref_rejected": -83.31040954589844, + "logps/rejected": -555.4446411132812, + "loss": 1.0726, + "margin_dpo/margin_mean": 217.7813720703125, + "margin_dpo/margin_std": 272.1701354980469, + "step": 366 + }, + { + "KL/chosen_KL_mean": -265.6231689453125, + "KL/mean": -356.1582336425781, + "KL/rejected_KL_mean": -446.69329833984375, + "KL/std": 205.83560180664062, + "epoch": 0.5389133627019089, + "fcm_dpo/beta": 0.0019680976402014494, + "fcm_dpo/delta": 0.04436366260051727, + "fcm_dpo/margin": 181.07012939453125, + "fcm_dpo/q_t": 0.4157208800315857, + "grad_norm": 19.829448699951172, + "learning_rate": 2.615458646614349e-07, + "logits/chosen": -0.4592265188694, + "logits/rejected": -0.4390316903591156, + "logps/chosen": -324.0285949707031, + "logps/ref_chosen": -58.405418395996094, + "logps/ref_rejected": -76.75132751464844, + "logps/rejected": -523.444580078125, + "loss": 1.1049, + "margin_dpo/margin_mean": 181.07012939453125, + "margin_dpo/margin_std": 216.0451202392578, + "step": 367 + }, + { + "KL/chosen_KL_mean": -253.53057861328125, + "KL/mean": -390.93402099609375, + "KL/rejected_KL_mean": -528.3375244140625, + "KL/std": 229.09591674804688, + "epoch": 0.540381791483113, + "fcm_dpo/beta": 0.0019493200816214085, + "fcm_dpo/delta": -0.13935965299606323, + "fcm_dpo/margin": 274.8069152832031, + "fcm_dpo/q_t": 0.3731291890144348, + "grad_norm": 39.253082275390625, + "learning_rate": 2.6026375718290083e-07, + "logits/chosen": -0.46787190437316895, + "logits/rejected": -0.47481292486190796, + "logps/chosen": -297.98309326171875, + "logps/ref_chosen": -44.452518463134766, + "logps/ref_rejected": -98.55526733398438, + "logps/rejected": -626.8927612304688, + "loss": 0.9579, + "margin_dpo/margin_mean": 274.8069152832031, + "margin_dpo/margin_std": 203.40640258789062, + "step": 368 + }, + { + "KL/chosen_KL_mean": -335.7747497558594, + "KL/mean": -411.5692138671875, + "KL/rejected_KL_mean": -487.3636474609375, + "KL/std": 234.81884765625, + "epoch": 0.5418502202643172, + "fcm_dpo/beta": 0.0019605052657425404, + "fcm_dpo/delta": 0.10399104654788971, + "fcm_dpo/margin": 151.58892822265625, + "fcm_dpo/q_t": 0.4309307336807251, + "grad_norm": 35.00344467163086, + "learning_rate": 2.589813792448196e-07, + "logits/chosen": -0.5002185702323914, + "logits/rejected": -0.4782012701034546, + "logps/chosen": -407.15625, + "logps/ref_chosen": -71.38150024414062, + "logps/ref_rejected": -91.29582214355469, + "logps/rejected": -578.6594848632812, + "loss": 1.1739, + "margin_dpo/margin_mean": 151.58892822265625, + "margin_dpo/margin_std": 259.8010559082031, + "step": 369 + }, + { + "KL/chosen_KL_mean": -357.89031982421875, + "KL/mean": -425.114990234375, + "KL/rejected_KL_mean": -492.33966064453125, + "KL/std": 241.58839416503906, + "epoch": 0.5433186490455213, + "fcm_dpo/beta": 0.0019837317522615194, + "fcm_dpo/delta": 0.1349019706249237, + "fcm_dpo/margin": 134.4493408203125, + "fcm_dpo/q_t": 0.4381292462348938, + "grad_norm": 32.33375549316406, + "learning_rate": 2.5769876463904263e-07, + "logits/chosen": -0.5110623836517334, + "logits/rejected": -0.5016048550605774, + "logps/chosen": -429.4978332519531, + "logps/ref_chosen": -71.60749816894531, + "logps/ref_rejected": -97.25978088378906, + "logps/rejected": -589.5994873046875, + "loss": 1.2033, + "margin_dpo/margin_mean": 134.4493408203125, + "margin_dpo/margin_std": 263.70660400390625, + "step": 370 + }, + { + "KL/chosen_KL_mean": -349.0395202636719, + "KL/mean": -454.69659423828125, + "KL/rejected_KL_mean": -560.3536987304688, + "KL/std": 257.37115478515625, + "epoch": 0.5447870778267254, + "fcm_dpo/beta": 0.0019879695028066635, + "fcm_dpo/delta": -0.020561737939715385, + "fcm_dpo/margin": 211.31417846679688, + "fcm_dpo/q_t": 0.40404176712036133, + "grad_norm": 22.81478500366211, + "learning_rate": 2.5641594716365744e-07, + "logits/chosen": -0.5306122303009033, + "logits/rejected": -0.5140925645828247, + "logps/chosen": -418.4540100097656, + "logps/ref_chosen": -69.41448974609375, + "logps/ref_rejected": -99.17217254638672, + "logps/rejected": -659.52587890625, + "loss": 1.0828, + "margin_dpo/margin_mean": 211.31417846679688, + "margin_dpo/margin_std": 280.86492919921875, + "step": 371 + }, + { + "KL/chosen_KL_mean": -343.93743896484375, + "KL/mean": -468.98333740234375, + "KL/rejected_KL_mean": -594.0292358398438, + "KL/std": 296.0002136230469, + "epoch": 0.5462555066079295, + "fcm_dpo/beta": 0.0019684848375618458, + "fcm_dpo/delta": -0.09475220739841461, + "fcm_dpo/margin": 250.091796875, + "fcm_dpo/q_t": 0.39228248596191406, + "grad_norm": 26.199745178222656, + "learning_rate": 2.551329606220976e-07, + "logits/chosen": -0.522529661655426, + "logits/rejected": -0.49960562586784363, + "logps/chosen": -405.75543212890625, + "logps/ref_chosen": -61.8179931640625, + "logps/ref_rejected": -78.53948974609375, + "logps/rejected": -672.5687255859375, + "loss": 1.0493, + "margin_dpo/margin_mean": 250.09181213378906, + "margin_dpo/margin_std": 338.2940673828125, + "step": 372 + }, + { + "KL/chosen_KL_mean": -372.826171875, + "KL/mean": -492.62713623046875, + "KL/rejected_KL_mean": -612.4281005859375, + "KL/std": 283.1507873535156, + "epoch": 0.5477239353891337, + "fcm_dpo/beta": 0.001959893386811018, + "fcm_dpo/delta": -0.07133302092552185, + "fcm_dpo/margin": 239.6019287109375, + "fcm_dpo/q_t": 0.3923265337944031, + "grad_norm": 32.04021072387695, + "learning_rate": 2.538498388222517e-07, + "logits/chosen": -0.5104029774665833, + "logits/rejected": -0.48797744512557983, + "logps/chosen": -437.04327392578125, + "logps/ref_chosen": -64.21713256835938, + "logps/ref_rejected": -85.95960998535156, + "logps/rejected": -698.3876953125, + "loss": 1.0425, + "margin_dpo/margin_mean": 239.6019287109375, + "margin_dpo/margin_std": 276.7124328613281, + "step": 373 + }, + { + "KL/chosen_KL_mean": -337.2335205078125, + "KL/mean": -444.0258483886719, + "KL/rejected_KL_mean": -550.8182373046875, + "KL/std": 301.60516357421875, + "epoch": 0.5491923641703378, + "fcm_dpo/beta": 0.001944505493156612, + "fcm_dpo/delta": -0.01592355966567993, + "fcm_dpo/margin": 213.58465576171875, + "fcm_dpo/q_t": 0.4093152582645416, + "grad_norm": 35.01904296875, + "learning_rate": 2.525666155755725e-07, + "logits/chosen": -0.5831848382949829, + "logits/rejected": -0.563025951385498, + "logps/chosen": -407.8836975097656, + "logps/ref_chosen": -70.65018463134766, + "logps/ref_rejected": -93.64016723632812, + "logps/rejected": -644.4583740234375, + "loss": 1.1086, + "margin_dpo/margin_mean": 213.5846710205078, + "margin_dpo/margin_std": 333.40771484375, + "step": 374 + }, + { + "KL/chosen_KL_mean": -351.09576416015625, + "KL/mean": -455.89691162109375, + "KL/rejected_KL_mean": -560.697998046875, + "KL/std": 252.77169799804688, + "epoch": 0.5506607929515418, + "fcm_dpo/beta": 0.00194136006757617, + "fcm_dpo/delta": -0.0074718669056892395, + "fcm_dpo/margin": 209.60220336914062, + "fcm_dpo/q_t": 0.4071958065032959, + "grad_norm": 37.43479537963867, + "learning_rate": 2.512833246961859e-07, + "logits/chosen": -0.5338048934936523, + "logits/rejected": -0.531802773475647, + "logps/chosen": -411.176025390625, + "logps/ref_chosen": -60.080223083496094, + "logps/ref_rejected": -88.93830871582031, + "logps/rejected": -649.6363525390625, + "loss": 1.1005, + "margin_dpo/margin_mean": 209.6022186279297, + "margin_dpo/margin_std": 296.93292236328125, + "step": 375 + }, + { + "KL/chosen_KL_mean": -351.7929992675781, + "KL/mean": -482.906494140625, + "KL/rejected_KL_mean": -614.02001953125, + "KL/std": 278.260009765625, + "epoch": 0.5521292217327459, + "fcm_dpo/beta": 0.0019302696455270052, + "fcm_dpo/delta": -0.10899202525615692, + "fcm_dpo/margin": 262.2270202636719, + "fcm_dpo/q_t": 0.38526269793510437, + "grad_norm": 26.96346092224121, + "learning_rate": 2.5e-07, + "logits/chosen": -0.5299756526947021, + "logits/rejected": -0.5187512040138245, + "logps/chosen": -414.45330810546875, + "logps/ref_chosen": -62.660308837890625, + "logps/ref_rejected": -105.52660369873047, + "logps/rejected": -719.546630859375, + "loss": 1.03, + "margin_dpo/margin_mean": 262.2270202636719, + "margin_dpo/margin_std": 318.892822265625, + "step": 376 + }, + { + "KL/chosen_KL_mean": -351.45703125, + "KL/mean": -473.6996765136719, + "KL/rejected_KL_mean": -595.9423828125, + "KL/std": 293.1138000488281, + "epoch": 0.55359765051395, + "fcm_dpo/beta": 0.001917961984872818, + "fcm_dpo/delta": -0.07053986191749573, + "fcm_dpo/margin": 244.4853515625, + "fcm_dpo/q_t": 0.39490044116973877, + "grad_norm": 32.56324005126953, + "learning_rate": 2.487166753038141e-07, + "logits/chosen": -0.4774741530418396, + "logits/rejected": -0.4759942889213562, + "logps/chosen": -405.9357604980469, + "logps/ref_chosen": -54.478736877441406, + "logps/ref_rejected": -98.70335388183594, + "logps/rejected": -694.645751953125, + "loss": 1.0543, + "margin_dpo/margin_mean": 244.4853515625, + "margin_dpo/margin_std": 313.40069580078125, + "step": 377 + }, + { + "KL/chosen_KL_mean": -332.24822998046875, + "KL/mean": -464.07470703125, + "KL/rejected_KL_mean": -595.9012451171875, + "KL/std": 259.45196533203125, + "epoch": 0.5550660792951542, + "fcm_dpo/beta": 0.001897640759125352, + "fcm_dpo/delta": -0.10289917886257172, + "fcm_dpo/margin": 263.6529541015625, + "fcm_dpo/q_t": 0.38391435146331787, + "grad_norm": 40.62555694580078, + "learning_rate": 2.4743338442442754e-07, + "logits/chosen": -0.5073676109313965, + "logits/rejected": -0.5229573249816895, + "logps/chosen": -377.2687683105469, + "logps/ref_chosen": -45.02053451538086, + "logps/ref_rejected": -88.0469741821289, + "logps/rejected": -683.9481811523438, + "loss": 1.0141, + "margin_dpo/margin_mean": 263.6529541015625, + "margin_dpo/margin_std": 282.84375, + "step": 378 + }, + { + "KL/chosen_KL_mean": -369.21722412109375, + "KL/mean": -490.90283203125, + "KL/rejected_KL_mean": -612.58837890625, + "KL/std": 263.14385986328125, + "epoch": 0.5565345080763583, + "fcm_dpo/beta": 0.0018807162996381521, + "fcm_dpo/delta": -0.059339869767427444, + "fcm_dpo/margin": 243.37112426757812, + "fcm_dpo/q_t": 0.3961556851863861, + "grad_norm": 41.72347640991211, + "learning_rate": 2.461501611777483e-07, + "logits/chosen": -0.5051323771476746, + "logits/rejected": -0.5261910557746887, + "logps/chosen": -422.39935302734375, + "logps/ref_chosen": -53.182098388671875, + "logps/ref_rejected": -114.3001708984375, + "logps/rejected": -726.8885498046875, + "loss": 1.0571, + "margin_dpo/margin_mean": 243.3711395263672, + "margin_dpo/margin_std": 304.7427978515625, + "step": 379 + }, + { + "KL/chosen_KL_mean": -356.805419921875, + "KL/mean": -492.5494384765625, + "KL/rejected_KL_mean": -628.29345703125, + "KL/std": 293.99835205078125, + "epoch": 0.5580029368575624, + "fcm_dpo/beta": 0.001867371378466487, + "fcm_dpo/delta": -0.10970066487789154, + "fcm_dpo/margin": 271.48809814453125, + "fcm_dpo/q_t": 0.3830481171607971, + "grad_norm": 29.717735290527344, + "learning_rate": 2.4486703937790243e-07, + "logits/chosen": -0.45388370752334595, + "logits/rejected": -0.480474054813385, + "logps/chosen": -408.158447265625, + "logps/ref_chosen": -51.3530387878418, + "logps/ref_rejected": -104.19169616699219, + "logps/rejected": -732.4851684570312, + "loss": 1.0195, + "margin_dpo/margin_mean": 271.4880676269531, + "margin_dpo/margin_std": 310.1554260253906, + "step": 380 + }, + { + "KL/chosen_KL_mean": -361.4216613769531, + "KL/mean": -462.06500244140625, + "KL/rejected_KL_mean": -562.7083129882812, + "KL/std": 248.3142852783203, + "epoch": 0.5594713656387665, + "fcm_dpo/beta": 0.001862676115706563, + "fcm_dpo/delta": 0.02552897296845913, + "fcm_dpo/margin": 201.28662109375, + "fcm_dpo/q_t": 0.4147588610649109, + "grad_norm": 27.22147560119629, + "learning_rate": 2.435840528363426e-07, + "logits/chosen": -0.4980400800704956, + "logits/rejected": -0.4761255383491516, + "logps/chosen": -419.2247314453125, + "logps/ref_chosen": -57.80306625366211, + "logps/ref_rejected": -79.21940612792969, + "logps/rejected": -641.927734375, + "loss": 1.1335, + "margin_dpo/margin_mean": 201.28662109375, + "margin_dpo/margin_std": 328.384033203125, + "step": 381 + }, + { + "KL/chosen_KL_mean": -360.59393310546875, + "KL/mean": -480.1690673828125, + "KL/rejected_KL_mean": -599.7442626953125, + "KL/std": 239.404296875, + "epoch": 0.5609397944199707, + "fcm_dpo/beta": 0.0018593824934214354, + "fcm_dpo/delta": -0.045690521597862244, + "fcm_dpo/margin": 239.15036010742188, + "fcm_dpo/q_t": 0.3968258202075958, + "grad_norm": 37.34941101074219, + "learning_rate": 2.4230123536095745e-07, + "logits/chosen": -0.5110651254653931, + "logits/rejected": -0.5134783983230591, + "logps/chosen": -426.61419677734375, + "logps/ref_chosen": -66.02030181884766, + "logps/ref_rejected": -110.71016693115234, + "logps/rejected": -710.4544677734375, + "loss": 1.0446, + "margin_dpo/margin_mean": 239.15036010742188, + "margin_dpo/margin_std": 259.5975646972656, + "step": 382 + }, + { + "KL/chosen_KL_mean": -368.52093505859375, + "KL/mean": -489.22088623046875, + "KL/rejected_KL_mean": -609.9208984375, + "KL/std": 261.40411376953125, + "epoch": 0.5624082232011748, + "fcm_dpo/beta": 0.0018501915037631989, + "fcm_dpo/delta": -0.04771873354911804, + "fcm_dpo/margin": 241.3999481201172, + "fcm_dpo/q_t": 0.39832523465156555, + "grad_norm": 27.045928955078125, + "learning_rate": 2.4101862075518037e-07, + "logits/chosen": -0.49125421047210693, + "logits/rejected": -0.4994109869003296, + "logps/chosen": -418.91241455078125, + "logps/ref_chosen": -50.39148712158203, + "logps/ref_rejected": -93.71589660644531, + "logps/rejected": -703.63671875, + "loss": 1.0802, + "margin_dpo/margin_mean": 241.39993286132812, + "margin_dpo/margin_std": 340.89404296875, + "step": 383 + }, + { + "KL/chosen_KL_mean": -365.1363525390625, + "KL/mean": -463.38427734375, + "KL/rejected_KL_mean": -561.6322021484375, + "KL/std": 249.699462890625, + "epoch": 0.5638766519823789, + "fcm_dpo/beta": 0.0018579517491161823, + "fcm_dpo/delta": 0.0352373830974102, + "fcm_dpo/margin": 196.4958038330078, + "fcm_dpo/q_t": 0.4140353202819824, + "grad_norm": 22.924989700317383, + "learning_rate": 2.397362428170992e-07, + "logits/chosen": -0.5382051467895508, + "logits/rejected": -0.529058575630188, + "logps/chosen": -417.1824645996094, + "logps/ref_chosen": -52.046104431152344, + "logps/ref_rejected": -85.76089477539062, + "logps/rejected": -647.39306640625, + "loss": 1.1015, + "margin_dpo/margin_mean": 196.4958038330078, + "margin_dpo/margin_std": 234.65902709960938, + "step": 384 + }, + { + "KL/chosen_KL_mean": -334.7351379394531, + "KL/mean": -454.896240234375, + "KL/rejected_KL_mean": -575.0573120117188, + "KL/std": 213.41387939453125, + "epoch": 0.5653450807635829, + "fcm_dpo/beta": 0.0018485202454030514, + "fcm_dpo/delta": -0.04525205120444298, + "fcm_dpo/margin": 240.32217407226562, + "fcm_dpo/q_t": 0.3952626585960388, + "grad_norm": 33.12544250488281, + "learning_rate": 2.3845413533856514e-07, + "logits/chosen": -0.531487762928009, + "logits/rejected": -0.4987449645996094, + "logps/chosen": -400.28729248046875, + "logps/ref_chosen": -65.55215454101562, + "logps/ref_rejected": -77.82792663574219, + "logps/rejected": -652.88525390625, + "loss": 1.0361, + "margin_dpo/margin_mean": 240.32217407226562, + "margin_dpo/margin_std": 236.79721069335938, + "step": 385 + }, + { + "KL/chosen_KL_mean": -352.7705078125, + "KL/mean": -480.95391845703125, + "KL/rejected_KL_mean": -609.1373291015625, + "KL/std": 269.5135498046875, + "epoch": 0.566813509544787, + "fcm_dpo/beta": 0.0018375307554379106, + "fcm_dpo/delta": -0.07277083396911621, + "fcm_dpo/margin": 256.3668212890625, + "fcm_dpo/q_t": 0.39270809292793274, + "grad_norm": 27.93240737915039, + "learning_rate": 2.3717233210430254e-07, + "logits/chosen": -0.5182179808616638, + "logits/rejected": -0.510960578918457, + "logps/chosen": -410.9923400878906, + "logps/ref_chosen": -58.22185516357422, + "logps/ref_rejected": -92.32742309570312, + "logps/rejected": -701.4647216796875, + "loss": 1.0448, + "margin_dpo/margin_mean": 256.3668212890625, + "margin_dpo/margin_std": 312.1371154785156, + "step": 386 + }, + { + "KL/chosen_KL_mean": -390.4794006347656, + "KL/mean": -495.60986328125, + "KL/rejected_KL_mean": -600.7403564453125, + "KL/std": 248.39511108398438, + "epoch": 0.5682819383259912, + "fcm_dpo/beta": 0.0018315855413675308, + "fcm_dpo/delta": 0.015043869614601135, + "fcm_dpo/margin": 210.26095581054688, + "fcm_dpo/q_t": 0.4094206690788269, + "grad_norm": 41.249908447265625, + "learning_rate": 2.3589086689101889e-07, + "logits/chosen": -0.6175644397735596, + "logits/rejected": -0.5995555520057678, + "logps/chosen": -456.89886474609375, + "logps/ref_chosen": -66.41944885253906, + "logps/ref_rejected": -92.16915893554688, + "logps/rejected": -692.9094848632812, + "loss": 1.0887, + "margin_dpo/margin_mean": 210.26095581054688, + "margin_dpo/margin_std": 248.58956909179688, + "step": 387 + }, + { + "KL/chosen_KL_mean": -377.8406677246094, + "KL/mean": -515.5494384765625, + "KL/rejected_KL_mean": -653.2581787109375, + "KL/std": 286.0020751953125, + "epoch": 0.5697503671071953, + "fcm_dpo/beta": 0.0018191644921898842, + "fcm_dpo/delta": -0.10386500507593155, + "fcm_dpo/margin": 275.41748046875, + "fcm_dpo/q_t": 0.3879193663597107, + "grad_norm": 30.190383911132812, + "learning_rate": 2.3460977346651428e-07, + "logits/chosen": -0.49368101358413696, + "logits/rejected": -0.5035091042518616, + "logps/chosen": -427.9701232910156, + "logps/ref_chosen": -50.129459381103516, + "logps/ref_rejected": -104.43305969238281, + "logps/rejected": -757.6912231445312, + "loss": 1.0263, + "margin_dpo/margin_mean": 275.41748046875, + "margin_dpo/margin_std": 328.736328125, + "step": 388 + }, + { + "KL/chosen_KL_mean": -415.5103454589844, + "KL/mean": -533.9041748046875, + "KL/rejected_KL_mean": -652.2980346679688, + "KL/std": 284.1199951171875, + "epoch": 0.5712187958883994, + "fcm_dpo/beta": 0.001811205642297864, + "fcm_dpo/delta": -0.029575031250715256, + "fcm_dpo/margin": 236.78768920898438, + "fcm_dpo/q_t": 0.40235158801078796, + "grad_norm": 33.09611892700195, + "learning_rate": 2.3332908558879177e-07, + "logits/chosen": -0.5412299633026123, + "logits/rejected": -0.5279366970062256, + "logps/chosen": -473.41693115234375, + "logps/ref_chosen": -57.906593322753906, + "logps/ref_rejected": -77.91454315185547, + "logps/rejected": -730.2125854492188, + "loss": 1.0798, + "margin_dpo/margin_mean": 236.78770446777344, + "margin_dpo/margin_std": 318.11456298828125, + "step": 389 + }, + { + "KL/chosen_KL_mean": -415.3945007324219, + "KL/mean": -532.8773193359375, + "KL/rejected_KL_mean": -650.360107421875, + "KL/std": 292.0013427734375, + "epoch": 0.5726872246696035, + "fcm_dpo/beta": 0.001804722473025322, + "fcm_dpo/delta": -0.024779455736279488, + "fcm_dpo/margin": 234.965576171875, + "fcm_dpo/q_t": 0.40768325328826904, + "grad_norm": 31.104320526123047, + "learning_rate": 2.320488370051681e-07, + "logits/chosen": -0.521344780921936, + "logits/rejected": -0.5126668810844421, + "logps/chosen": -464.62042236328125, + "logps/ref_chosen": -49.22591781616211, + "logps/ref_rejected": -85.5281982421875, + "logps/rejected": -735.8883056640625, + "loss": 1.1072, + "margin_dpo/margin_mean": 234.965576171875, + "margin_dpo/margin_std": 377.9402160644531, + "step": 390 + }, + { + "KL/chosen_KL_mean": -402.7524719238281, + "KL/mean": -475.3613586425781, + "KL/rejected_KL_mean": -547.97021484375, + "KL/std": 272.24322509765625, + "epoch": 0.5741556534508077, + "fcm_dpo/beta": 0.0018261673394590616, + "fcm_dpo/delta": 0.13658833503723145, + "fcm_dpo/margin": 145.21774291992188, + "fcm_dpo/q_t": 0.43881237506866455, + "grad_norm": 44.82551193237305, + "learning_rate": 2.3076906145138405e-07, + "logits/chosen": -0.5299597978591919, + "logits/rejected": -0.5175088047981262, + "logps/chosen": -467.0821228027344, + "logps/ref_chosen": -64.32965087890625, + "logps/ref_rejected": -86.73820495605469, + "logps/rejected": -634.7084350585938, + "loss": 1.1992, + "margin_dpo/margin_mean": 145.2177276611328, + "margin_dpo/margin_std": 272.92193603515625, + "step": 391 + }, + { + "KL/chosen_KL_mean": -344.9163513183594, + "KL/mean": -484.80401611328125, + "KL/rejected_KL_mean": -624.6917114257812, + "KL/std": 282.89739990234375, + "epoch": 0.5756240822320118, + "fcm_dpo/beta": 0.0018200704362243414, + "fcm_dpo/delta": -0.11199337244033813, + "fcm_dpo/margin": 279.7752990722656, + "fcm_dpo/q_t": 0.383048415184021, + "grad_norm": 29.697546005249023, + "learning_rate": 2.294897926507156e-07, + "logits/chosen": -0.5169392824172974, + "logits/rejected": -0.5098834037780762, + "logps/chosen": -398.42034912109375, + "logps/ref_chosen": -53.50397872924805, + "logps/ref_rejected": -102.34584045410156, + "logps/rejected": -727.0375366210938, + "loss": 0.9991, + "margin_dpo/margin_mean": 279.77532958984375, + "margin_dpo/margin_std": 275.7098083496094, + "step": 392 + }, + { + "KL/chosen_KL_mean": -343.71734619140625, + "KL/mean": -456.7199401855469, + "KL/rejected_KL_mean": -569.7225341796875, + "KL/std": 284.97540283203125, + "epoch": 0.5770925110132159, + "fcm_dpo/beta": 0.0018092221580445766, + "fcm_dpo/delta": -0.009102771058678627, + "fcm_dpo/margin": 226.00515747070312, + "fcm_dpo/q_t": 0.41020962595939636, + "grad_norm": 47.57746124267578, + "learning_rate": 2.2821106431308543e-07, + "logits/chosen": -0.5029352307319641, + "logits/rejected": -0.504021406173706, + "logps/chosen": -390.1912841796875, + "logps/ref_chosen": -46.473915100097656, + "logps/ref_rejected": -71.96885681152344, + "logps/rejected": -641.69140625, + "loss": 1.106, + "margin_dpo/margin_mean": 226.00515747070312, + "margin_dpo/margin_std": 352.2773742675781, + "step": 393 + }, + { + "KL/chosen_KL_mean": -391.92578125, + "KL/mean": -508.56024169921875, + "KL/rejected_KL_mean": -625.1947021484375, + "KL/std": 295.7176818847656, + "epoch": 0.57856093979442, + "fcm_dpo/beta": 0.0018048053607344627, + "fcm_dpo/delta": -0.02154848352074623, + "fcm_dpo/margin": 233.2689208984375, + "fcm_dpo/q_t": 0.4036337435245514, + "grad_norm": 27.146669387817383, + "learning_rate": 2.2693291013417452e-07, + "logits/chosen": -0.5260942578315735, + "logits/rejected": -0.524019181728363, + "logps/chosen": -444.83734130859375, + "logps/ref_chosen": -52.91154861450195, + "logps/ref_rejected": -90.8226318359375, + "logps/rejected": -716.017333984375, + "loss": 1.0749, + "margin_dpo/margin_mean": 233.2689208984375, + "margin_dpo/margin_std": 294.8382568359375, + "step": 394 + }, + { + "KL/chosen_KL_mean": -382.04046630859375, + "KL/mean": -507.2308654785156, + "KL/rejected_KL_mean": -632.4212646484375, + "KL/std": 293.729736328125, + "epoch": 0.580029368575624, + "fcm_dpo/beta": 0.001797014381736517, + "fcm_dpo/delta": -0.051209770143032074, + "fcm_dpo/margin": 250.38079833984375, + "fcm_dpo/q_t": 0.39935189485549927, + "grad_norm": 38.09134292602539, + "learning_rate": 2.2565536379453404e-07, + "logits/chosen": -0.5735211372375488, + "logits/rejected": -0.5697096586227417, + "logps/chosen": -444.5865478515625, + "logps/ref_chosen": -62.546112060546875, + "logps/ref_rejected": -83.78262329101562, + "logps/rejected": -716.203857421875, + "loss": 1.0749, + "margin_dpo/margin_mean": 250.38082885742188, + "margin_dpo/margin_std": 346.9578857421875, + "step": 395 + }, + { + "KL/chosen_KL_mean": -390.10418701171875, + "KL/mean": -504.8359375, + "KL/rejected_KL_mean": -619.5677490234375, + "KL/std": 280.2290344238281, + "epoch": 0.5814977973568282, + "fcm_dpo/beta": 0.00179403240326792, + "fcm_dpo/delta": -0.01193785946816206, + "fcm_dpo/margin": 229.46353149414062, + "fcm_dpo/q_t": 0.4044559597969055, + "grad_norm": 26.76568031311035, + "learning_rate": 2.2437845895869825e-07, + "logits/chosen": -0.5892349481582642, + "logits/rejected": -0.5674149394035339, + "logps/chosen": -459.10009765625, + "logps/ref_chosen": -68.99594116210938, + "logps/ref_rejected": -88.64665985107422, + "logps/rejected": -708.21435546875, + "loss": 1.0683, + "margin_dpo/margin_mean": 229.4635467529297, + "margin_dpo/margin_std": 264.5689392089844, + "step": 396 + }, + { + "KL/chosen_KL_mean": -362.35675048828125, + "KL/mean": -515.5724487304688, + "KL/rejected_KL_mean": -668.7880859375, + "KL/std": 288.4814453125, + "epoch": 0.5829662261380323, + "fcm_dpo/beta": 0.0017716542351990938, + "fcm_dpo/delta": -0.14731790125370026, + "fcm_dpo/margin": 306.431396484375, + "fcm_dpo/q_t": 0.3749983310699463, + "grad_norm": 41.37638854980469, + "learning_rate": 2.2310222927429716e-07, + "logits/chosen": -0.5667599439620972, + "logits/rejected": -0.5667216181755066, + "logps/chosen": -423.6339111328125, + "logps/ref_chosen": -61.27716827392578, + "logps/ref_rejected": -103.11612701416016, + "logps/rejected": -771.9042358398438, + "loss": 0.9805, + "margin_dpo/margin_mean": 306.4313659667969, + "margin_dpo/margin_std": 299.587646484375, + "step": 397 + }, + { + "KL/chosen_KL_mean": -423.58306884765625, + "KL/mean": -554.8473510742188, + "KL/rejected_KL_mean": -686.1116943359375, + "KL/std": 320.2618713378906, + "epoch": 0.5844346549192364, + "fcm_dpo/beta": 0.0017602917505428195, + "fcm_dpo/delta": -0.0635901689529419, + "fcm_dpo/margin": 262.528564453125, + "fcm_dpo/q_t": 0.3985213041305542, + "grad_norm": 32.223777770996094, + "learning_rate": 2.2182670837116972e-07, + "logits/chosen": -0.6214488744735718, + "logits/rejected": -0.6171579957008362, + "logps/chosen": -491.734619140625, + "logps/ref_chosen": -68.15155029296875, + "logps/ref_rejected": -108.52360534667969, + "logps/rejected": -794.63525390625, + "loss": 1.0673, + "margin_dpo/margin_mean": 262.528564453125, + "margin_dpo/margin_std": 365.87701416015625, + "step": 398 + }, + { + "KL/chosen_KL_mean": -380.365478515625, + "KL/mean": -508.25189208984375, + "KL/rejected_KL_mean": -636.1383056640625, + "KL/std": 293.50164794921875, + "epoch": 0.5859030837004405, + "fcm_dpo/beta": 0.0017485737334936857, + "fcm_dpo/delta": -0.048456672579050064, + "fcm_dpo/margin": 255.77288818359375, + "fcm_dpo/q_t": 0.40013986825942993, + "grad_norm": 38.44267272949219, + "learning_rate": 2.2055192986047804e-07, + "logits/chosen": -0.5989868640899658, + "logits/rejected": -0.5554165840148926, + "logps/chosen": -441.2552795410156, + "logps/ref_chosen": -60.889801025390625, + "logps/ref_rejected": -77.965576171875, + "logps/rejected": -714.1038818359375, + "loss": 1.08, + "margin_dpo/margin_mean": 255.77285766601562, + "margin_dpo/margin_std": 361.84649658203125, + "step": 399 + }, + { + "KL/chosen_KL_mean": -368.0055236816406, + "KL/mean": -543.0501098632812, + "KL/rejected_KL_mean": -718.0947265625, + "KL/std": 286.2700500488281, + "epoch": 0.5873715124816447, + "fcm_dpo/beta": 0.0017187241464853287, + "fcm_dpo/delta": -0.20846638083457947, + "fcm_dpo/margin": 350.08917236328125, + "fcm_dpo/q_t": 0.3612917959690094, + "grad_norm": 26.391523361206055, + "learning_rate": 2.192779273338215e-07, + "logits/chosen": -0.5936537981033325, + "logits/rejected": -0.5919471383094788, + "logps/chosen": -431.64910888671875, + "logps/ref_chosen": -63.64359664916992, + "logps/ref_rejected": -105.252685546875, + "logps/rejected": -823.347412109375, + "loss": 0.9419, + "margin_dpo/margin_mean": 350.08917236328125, + "margin_dpo/margin_std": 319.36767578125, + "step": 400 + }, + { + "KL/chosen_KL_mean": -435.6270446777344, + "KL/mean": -533.0802612304688, + "KL/rejected_KL_mean": -630.533447265625, + "KL/std": 307.6553039550781, + "epoch": 0.5888399412628488, + "fcm_dpo/beta": 0.0017190496437251568, + "fcm_dpo/delta": 0.06607242673635483, + "fcm_dpo/margin": 194.9063720703125, + "fcm_dpo/q_t": 0.426999032497406, + "grad_norm": 35.03347396850586, + "learning_rate": 2.1800473436235136e-07, + "logits/chosen": -0.591301441192627, + "logits/rejected": -0.5788199305534363, + "logps/chosen": -492.79010009765625, + "logps/ref_chosen": -57.16303253173828, + "logps/ref_rejected": -83.79249572753906, + "logps/rejected": -714.325927734375, + "loss": 1.1903, + "margin_dpo/margin_mean": 194.9063720703125, + "margin_dpo/margin_std": 405.22808837890625, + "step": 401 + }, + { + "KL/chosen_KL_mean": -349.45294189453125, + "KL/mean": -544.983642578125, + "KL/rejected_KL_mean": -740.514404296875, + "KL/std": 371.795654296875, + "epoch": 0.5903083700440529, + "fcm_dpo/beta": 0.001689492492005229, + "fcm_dpo/delta": -0.26964718103408813, + "fcm_dpo/margin": 391.06146240234375, + "fcm_dpo/q_t": 0.3538634181022644, + "grad_norm": 27.208879470825195, + "learning_rate": 2.1673238449588665e-07, + "logits/chosen": -0.6506165266036987, + "logits/rejected": -0.6465029120445251, + "logps/chosen": -400.1933288574219, + "logps/ref_chosen": -50.74037170410156, + "logps/ref_rejected": -81.0460433959961, + "logps/rejected": -821.5604248046875, + "loss": 0.917, + "margin_dpo/margin_mean": 391.06146240234375, + "margin_dpo/margin_std": 385.4092712402344, + "step": 402 + }, + { + "KL/chosen_KL_mean": -376.1459655761719, + "KL/mean": -522.2297973632812, + "KL/rejected_KL_mean": -668.3135375976562, + "KL/std": 303.7095031738281, + "epoch": 0.591776798825257, + "fcm_dpo/beta": 0.0016665069852024317, + "fcm_dpo/delta": -0.08910196274518967, + "fcm_dpo/margin": 292.1676330566406, + "fcm_dpo/q_t": 0.388282835483551, + "grad_norm": 30.522640228271484, + "learning_rate": 2.154609112620295e-07, + "logits/chosen": -0.6762701869010925, + "logits/rejected": -0.6811692714691162, + "logps/chosen": -423.29327392578125, + "logps/ref_chosen": -47.14731216430664, + "logps/ref_rejected": -77.2666015625, + "logps/rejected": -745.5802001953125, + "loss": 1.0256, + "margin_dpo/margin_mean": 292.1676025390625, + "margin_dpo/margin_std": 321.37841796875, + "step": 403 + }, + { + "KL/chosen_KL_mean": -453.879638671875, + "KL/mean": -607.360107421875, + "KL/rejected_KL_mean": -760.840576171875, + "KL/std": 370.8985595703125, + "epoch": 0.593245227606461, + "fcm_dpo/beta": 0.0016497753094881773, + "fcm_dpo/delta": -0.1091829165816307, + "fcm_dpo/margin": 306.96087646484375, + "fcm_dpo/q_t": 0.3911134600639343, + "grad_norm": 31.827224731445312, + "learning_rate": 2.1419034816528218e-07, + "logits/chosen": -0.6803830862045288, + "logits/rejected": -0.6808692216873169, + "logps/chosen": -501.75494384765625, + "logps/ref_chosen": -47.875274658203125, + "logps/ref_rejected": -77.15499877929688, + "logps/rejected": -837.9955444335938, + "loss": 1.081, + "margin_dpo/margin_mean": 306.96087646484375, + "margin_dpo/margin_std": 484.5469970703125, + "step": 404 + }, + { + "KL/chosen_KL_mean": -506.3373718261719, + "KL/mean": -623.0357666015625, + "KL/rejected_KL_mean": -739.7340698242188, + "KL/std": 341.96575927734375, + "epoch": 0.5947136563876652, + "fcm_dpo/beta": 0.0016373193357139826, + "fcm_dpo/delta": -0.09679268300533295, + "fcm_dpo/margin": 233.39666748046875, + "fcm_dpo/q_t": 0.4155503511428833, + "grad_norm": 44.04417419433594, + "learning_rate": 2.129207286861638e-07, + "logits/chosen": -0.6529548168182373, + "logits/rejected": -0.6453539729118347, + "logps/chosen": -571.500244140625, + "logps/ref_chosen": -65.16290283203125, + "logps/ref_rejected": -87.18678283691406, + "logps/rejected": -826.9208984375, + "loss": 1.1604, + "margin_dpo/margin_mean": 233.3966827392578, + "margin_dpo/margin_std": 420.79632568359375, + "step": 405 + }, + { + "KL/chosen_KL_mean": -504.4444580078125, + "KL/mean": -670.9434814453125, + "KL/rejected_KL_mean": -837.4425048828125, + "KL/std": 409.6977233886719, + "epoch": 0.5961820851688693, + "fcm_dpo/beta": 0.0016175368800759315, + "fcm_dpo/delta": -0.14244601130485535, + "fcm_dpo/margin": 332.998046875, + "fcm_dpo/q_t": 0.38683146238327026, + "grad_norm": 31.830657958984375, + "learning_rate": 2.1165208628032861e-07, + "logits/chosen": -0.7117749452590942, + "logits/rejected": -0.7342991232872009, + "logps/chosen": -554.1852416992188, + "logps/ref_chosen": -49.740814208984375, + "logps/ref_rejected": -92.07862854003906, + "logps/rejected": -929.5211181640625, + "loss": 1.041, + "margin_dpo/margin_mean": 332.998046875, + "margin_dpo/margin_std": 462.95513916015625, + "step": 406 + }, + { + "KL/chosen_KL_mean": -487.5089111328125, + "KL/mean": -582.2493896484375, + "KL/rejected_KL_mean": -676.9898681640625, + "KL/std": 341.89263916015625, + "epoch": 0.5976505139500734, + "fcm_dpo/beta": 0.0016208180459216237, + "fcm_dpo/delta": 0.0940016433596611, + "fcm_dpo/margin": 189.48098754882812, + "fcm_dpo/q_t": 0.42886239290237427, + "grad_norm": 40.08480453491211, + "learning_rate": 2.1038445437768375e-07, + "logits/chosen": -0.7205852270126343, + "logits/rejected": -0.6887321472167969, + "logps/chosen": -543.839599609375, + "logps/ref_chosen": -56.33069610595703, + "logps/ref_rejected": -77.51209259033203, + "logps/rejected": -754.501953125, + "loss": 1.2139, + "margin_dpo/margin_mean": 189.48097229003906, + "margin_dpo/margin_std": 426.8612060546875, + "step": 407 + }, + { + "KL/chosen_KL_mean": -488.0704345703125, + "KL/mean": -594.4813232421875, + "KL/rejected_KL_mean": -700.8922729492188, + "KL/std": 310.1326904296875, + "epoch": 0.5991189427312775, + "fcm_dpo/beta": 0.0016301569994539022, + "fcm_dpo/delta": 0.05386827886104584, + "fcm_dpo/margin": 212.82177734375, + "fcm_dpo/q_t": 0.4206947088241577, + "grad_norm": 41.006221771240234, + "learning_rate": 2.0911786638150872e-07, + "logits/chosen": -0.6842703819274902, + "logits/rejected": -0.6612120866775513, + "logps/chosen": -557.8597412109375, + "logps/ref_chosen": -69.789306640625, + "logps/ref_rejected": -90.09693908691406, + "logps/rejected": -790.9891967773438, + "loss": 1.1492, + "margin_dpo/margin_mean": 212.82179260253906, + "margin_dpo/margin_std": 353.79248046875, + "step": 408 + }, + { + "KL/chosen_KL_mean": -453.1000671386719, + "KL/mean": -564.703125, + "KL/rejected_KL_mean": -676.30615234375, + "KL/std": 381.3478088378906, + "epoch": 0.6005873715124816, + "fcm_dpo/beta": 0.0016393279656767845, + "fcm_dpo/delta": 0.03426938131451607, + "fcm_dpo/margin": 223.20606994628906, + "fcm_dpo/q_t": 0.41481828689575195, + "grad_norm": 59.193416595458984, + "learning_rate": 2.0785235566757517e-07, + "logits/chosen": -0.6385992765426636, + "logits/rejected": -0.6249116063117981, + "logps/chosen": -520.41748046875, + "logps/ref_chosen": -67.31744384765625, + "logps/ref_rejected": -84.904296875, + "logps/rejected": -761.21044921875, + "loss": 1.1496, + "margin_dpo/margin_mean": 223.20608520507812, + "margin_dpo/margin_std": 382.4623107910156, + "step": 409 + }, + { + "KL/chosen_KL_mean": -416.6119384765625, + "KL/mean": -551.2454833984375, + "KL/rejected_KL_mean": -685.8790893554688, + "KL/std": 340.37841796875, + "epoch": 0.6020558002936858, + "fcm_dpo/beta": 0.0016306547913700342, + "fcm_dpo/delta": -0.039969172328710556, + "fcm_dpo/margin": 269.26708984375, + "fcm_dpo/q_t": 0.39879608154296875, + "grad_norm": 33.50337219238281, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": -0.6794567108154297, + "logits/rejected": -0.6865516901016235, + "logps/chosen": -468.0772705078125, + "logps/ref_chosen": -51.465354919433594, + "logps/ref_rejected": -83.198974609375, + "logps/rejected": -769.0780639648438, + "loss": 1.0792, + "margin_dpo/margin_mean": 269.26708984375, + "margin_dpo/margin_std": 368.21893310546875, + "step": 410 + }, + { + "KL/chosen_KL_mean": -423.6837158203125, + "KL/mean": -541.8031616210938, + "KL/rejected_KL_mean": -659.922607421875, + "KL/std": 336.0458984375, + "epoch": 0.6035242290748899, + "fcm_dpo/beta": 0.0016164029948413372, + "fcm_dpo/delta": -0.07978048175573349, + "fcm_dpo/margin": 236.23883056640625, + "fcm_dpo/q_t": 0.4166368842124939, + "grad_norm": 45.770408630371094, + "learning_rate": 2.0532469944670343e-07, + "logits/chosen": -0.6459128260612488, + "logits/rejected": -0.6529507637023926, + "logps/chosen": -475.9909973144531, + "logps/ref_chosen": -52.30727005004883, + "logps/ref_rejected": -80.69495391845703, + "logps/rejected": -740.6175537109375, + "loss": 1.1373, + "margin_dpo/margin_mean": 236.23883056640625, + "margin_dpo/margin_std": 390.1459655761719, + "step": 411 + }, + { + "KL/chosen_KL_mean": -404.8624267578125, + "KL/mean": -531.0133056640625, + "KL/rejected_KL_mean": -657.1642456054688, + "KL/std": 315.80413818359375, + "epoch": 0.604992657856094, + "fcm_dpo/beta": 0.0016159769147634506, + "fcm_dpo/delta": -0.007876180112361908, + "fcm_dpo/margin": 252.3018341064453, + "fcm_dpo/q_t": 0.40613073110580444, + "grad_norm": 43.324363708496094, + "learning_rate": 2.0406262054585738e-07, + "logits/chosen": -0.6817220449447632, + "logits/rejected": -0.7102159261703491, + "logps/chosen": -458.00653076171875, + "logps/ref_chosen": -53.144126892089844, + "logps/ref_rejected": -100.0608139038086, + "logps/rejected": -757.2250366210938, + "loss": 1.0908, + "margin_dpo/margin_mean": 252.3018341064453, + "margin_dpo/margin_std": 339.39776611328125, + "step": 412 + }, + { + "KL/chosen_KL_mean": -445.6668701171875, + "KL/mean": -565.676025390625, + "KL/rejected_KL_mean": -685.6851806640625, + "KL/std": 343.90045166015625, + "epoch": 0.6064610866372981, + "fcm_dpo/beta": 0.0016158015932887793, + "fcm_dpo/delta": 0.012414616532623768, + "fcm_dpo/margin": 240.018310546875, + "fcm_dpo/q_t": 0.4103829562664032, + "grad_norm": 48.77497482299805, + "learning_rate": 2.0280175213768205e-07, + "logits/chosen": -0.581852912902832, + "logits/rejected": -0.5829395055770874, + "logps/chosen": -507.24884033203125, + "logps/ref_chosen": -61.58196258544922, + "logps/ref_rejected": -99.47340393066406, + "logps/rejected": -785.1585693359375, + "loss": 1.1251, + "margin_dpo/margin_mean": 240.01829528808594, + "margin_dpo/margin_std": 382.1087646484375, + "step": 413 + }, + { + "KL/chosen_KL_mean": -410.1985778808594, + "KL/mean": -545.940673828125, + "KL/rejected_KL_mean": -681.6827392578125, + "KL/std": 313.23284912109375, + "epoch": 0.6079295154185022, + "fcm_dpo/beta": 0.0016170567832887173, + "fcm_dpo/delta": -0.04011237248778343, + "fcm_dpo/margin": 271.4841613769531, + "fcm_dpo/q_t": 0.3984266519546509, + "grad_norm": 41.59056091308594, + "learning_rate": 2.0154212744723247e-07, + "logits/chosen": -0.5111421346664429, + "logits/rejected": -0.5018640160560608, + "logps/chosen": -456.830078125, + "logps/ref_chosen": -46.63148498535156, + "logps/ref_rejected": -87.64653015136719, + "logps/rejected": -769.3292236328125, + "loss": 1.0743, + "margin_dpo/margin_mean": 271.4841613769531, + "margin_dpo/margin_std": 356.08306884765625, + "step": 414 + }, + { + "KL/chosen_KL_mean": -406.1727294921875, + "KL/mean": -505.6904296875, + "KL/rejected_KL_mean": -605.2080688476562, + "KL/std": 278.98065185546875, + "epoch": 0.6093979441997063, + "fcm_dpo/beta": 0.0016174819320440292, + "fcm_dpo/delta": 0.07935836911201477, + "fcm_dpo/margin": 199.03533935546875, + "fcm_dpo/q_t": 0.4250248372554779, + "grad_norm": 32.30403518676758, + "learning_rate": 2.002837796667909e-07, + "logits/chosen": -0.5725187063217163, + "logits/rejected": -0.5640978813171387, + "logps/chosen": -484.791015625, + "logps/ref_chosen": -78.6182861328125, + "logps/ref_rejected": -100.47752380371094, + "logps/rejected": -705.685546875, + "loss": 1.1467, + "margin_dpo/margin_mean": 199.0353240966797, + "margin_dpo/margin_std": 299.36395263671875, + "step": 415 + }, + { + "KL/chosen_KL_mean": -368.3544616699219, + "KL/mean": -525.67529296875, + "KL/rejected_KL_mean": -682.99609375, + "KL/std": 330.7052001953125, + "epoch": 0.6108663729809104, + "fcm_dpo/beta": 0.001612334861420095, + "fcm_dpo/delta": -0.11002575606107712, + "fcm_dpo/margin": 314.6416015625, + "fcm_dpo/q_t": 0.38389724493026733, + "grad_norm": 30.73332405090332, + "learning_rate": 1.990267419549914e-07, + "logits/chosen": -0.5767337083816528, + "logits/rejected": -0.577792763710022, + "logps/chosen": -426.63360595703125, + "logps/ref_chosen": -58.27912521362305, + "logps/ref_rejected": -90.56871795654297, + "logps/rejected": -773.5648193359375, + "loss": 1.011, + "margin_dpo/margin_mean": 314.6416015625, + "margin_dpo/margin_std": 339.9158020019531, + "step": 416 + }, + { + "KL/chosen_KL_mean": -364.384765625, + "KL/mean": -508.7856140136719, + "KL/rejected_KL_mean": -653.1865234375, + "KL/std": 298.44720458984375, + "epoch": 0.6123348017621145, + "fcm_dpo/beta": 0.001599437789991498, + "fcm_dpo/delta": -0.06335951387882233, + "fcm_dpo/margin": 288.8017272949219, + "fcm_dpo/q_t": 0.391997754573822, + "grad_norm": 47.862770080566406, + "learning_rate": 1.9777104743594686e-07, + "logits/chosen": -0.5380935668945312, + "logits/rejected": -0.5061866641044617, + "logps/chosen": -414.58349609375, + "logps/ref_chosen": -50.1987190246582, + "logps/ref_rejected": -68.15184020996094, + "logps/rejected": -721.3383178710938, + "loss": 1.0336, + "margin_dpo/margin_mean": 288.8017272949219, + "margin_dpo/margin_std": 306.3311462402344, + "step": 417 + }, + { + "KL/chosen_KL_mean": -390.33111572265625, + "KL/mean": -528.9058837890625, + "KL/rejected_KL_mean": -667.4805908203125, + "KL/std": 329.7533874511719, + "epoch": 0.6138032305433186, + "fcm_dpo/beta": 0.001596780028194189, + "fcm_dpo/delta": -0.04398631677031517, + "fcm_dpo/margin": 277.1495361328125, + "fcm_dpo/q_t": 0.4014670252799988, + "grad_norm": 24.271198272705078, + "learning_rate": 1.965167291983757e-07, + "logits/chosen": -0.6195993423461914, + "logits/rejected": -0.5918015241622925, + "logps/chosen": -472.3095703125, + "logps/ref_chosen": -81.97846984863281, + "logps/ref_rejected": -104.69148254394531, + "logps/rejected": -772.172119140625, + "loss": 1.0803, + "margin_dpo/margin_mean": 277.1495361328125, + "margin_dpo/margin_std": 378.1399841308594, + "step": 418 + }, + { + "KL/chosen_KL_mean": -359.47064208984375, + "KL/mean": -516.6668701171875, + "KL/rejected_KL_mean": -673.8631591796875, + "KL/std": 296.4808349609375, + "epoch": 0.6152716593245228, + "fcm_dpo/beta": 0.0015776680083945394, + "fcm_dpo/delta": -0.09841260313987732, + "fcm_dpo/margin": 314.3925476074219, + "fcm_dpo/q_t": 0.38518738746643066, + "grad_norm": 39.38912582397461, + "learning_rate": 1.9526382029472988e-07, + "logits/chosen": -0.5384039282798767, + "logits/rejected": -0.5316938161849976, + "logps/chosen": -412.4192810058594, + "logps/ref_chosen": -52.948646545410156, + "logps/ref_rejected": -91.58309936523438, + "logps/rejected": -765.4462890625, + "loss": 1.0206, + "margin_dpo/margin_mean": 314.39251708984375, + "margin_dpo/margin_std": 347.848876953125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -473.1541748046875, + "KL/mean": -576.36474609375, + "KL/rejected_KL_mean": -679.5751953125, + "KL/std": 321.3580322265625, + "epoch": 0.6167400881057269, + "fcm_dpo/beta": 0.0015831240452826023, + "fcm_dpo/delta": 0.0743355080485344, + "fcm_dpo/margin": 206.4210968017578, + "fcm_dpo/q_t": 0.4267392158508301, + "grad_norm": 55.58370590209961, + "learning_rate": 1.9401235374032425e-07, + "logits/chosen": -0.5968104600906372, + "logits/rejected": -0.5501741170883179, + "logps/chosen": -550.924072265625, + "logps/ref_chosen": -77.7699203491211, + "logps/ref_rejected": -69.31985473632812, + "logps/rejected": -748.8951416015625, + "loss": 1.1855, + "margin_dpo/margin_mean": 206.42111206054688, + "margin_dpo/margin_std": 412.8056640625, + "step": 420 + }, + { + "KL/chosen_KL_mean": -396.0514831542969, + "KL/mean": -511.7779235839844, + "KL/rejected_KL_mean": -627.5043334960938, + "KL/std": 307.17010498046875, + "epoch": 0.618208516886931, + "fcm_dpo/beta": 0.0015912302769720554, + "fcm_dpo/delta": 0.03213408216834068, + "fcm_dpo/margin": 231.45286560058594, + "fcm_dpo/q_t": 0.41308170557022095, + "grad_norm": 37.79745101928711, + "learning_rate": 1.9276236251246653e-07, + "logits/chosen": -0.6047526001930237, + "logits/rejected": -0.5907719135284424, + "logps/chosen": -449.8173522949219, + "logps/ref_chosen": -53.765865325927734, + "logps/ref_rejected": -89.28144836425781, + "logps/rejected": -716.7857666015625, + "loss": 1.1119, + "margin_dpo/margin_mean": 231.45286560058594, + "margin_dpo/margin_std": 305.76007080078125, + "step": 421 + }, + { + "KL/chosen_KL_mean": -444.47344970703125, + "KL/mean": -577.7868041992188, + "KL/rejected_KL_mean": -711.10009765625, + "KL/std": 318.5824890136719, + "epoch": 0.6196769456681351, + "fcm_dpo/beta": 0.0015860656276345253, + "fcm_dpo/delta": -0.02341364696621895, + "fcm_dpo/margin": 266.6266784667969, + "fcm_dpo/q_t": 0.4029381275177002, + "grad_norm": 37.30603790283203, + "learning_rate": 1.9151387954958792e-07, + "logits/chosen": -0.6128122806549072, + "logits/rejected": -0.6130838394165039, + "logps/chosen": -513.1072387695312, + "logps/ref_chosen": -68.6337661743164, + "logps/ref_rejected": -87.86351013183594, + "logps/rejected": -798.963623046875, + "loss": 1.09, + "margin_dpo/margin_mean": 266.62664794921875, + "margin_dpo/margin_std": 376.72772216796875, + "step": 422 + }, + { + "KL/chosen_KL_mean": -442.69097900390625, + "KL/mean": -593.7772216796875, + "KL/rejected_KL_mean": -744.863525390625, + "KL/std": 316.8233947753906, + "epoch": 0.6211453744493393, + "fcm_dpo/beta": 0.0015762131661176682, + "fcm_dpo/delta": -0.07819212973117828, + "fcm_dpo/margin": 302.17254638671875, + "fcm_dpo/q_t": 0.39091968536376953, + "grad_norm": 35.244747161865234, + "learning_rate": 1.902669377503756e-07, + "logits/chosen": -0.6149768829345703, + "logits/rejected": -0.6239042282104492, + "logps/chosen": -497.6812744140625, + "logps/ref_chosen": -54.99030303955078, + "logps/ref_rejected": -86.30654907226562, + "logps/rejected": -831.1700439453125, + "loss": 1.0466, + "margin_dpo/margin_mean": 302.17254638671875, + "margin_dpo/margin_std": 374.8946533203125, + "step": 423 + }, + { + "KL/chosen_KL_mean": -408.0943908691406, + "KL/mean": -545.8341064453125, + "KL/rejected_KL_mean": -683.57373046875, + "KL/std": 307.42901611328125, + "epoch": 0.6226138032305433, + "fcm_dpo/beta": 0.0015672901645302773, + "fcm_dpo/delta": -0.03262971714138985, + "fcm_dpo/margin": 275.4793395996094, + "fcm_dpo/q_t": 0.40227210521698, + "grad_norm": 37.73208236694336, + "learning_rate": 1.890215699729057e-07, + "logits/chosen": -0.6178228855133057, + "logits/rejected": -0.5884814262390137, + "logps/chosen": -464.1063232421875, + "logps/ref_chosen": -56.01192092895508, + "logps/ref_rejected": -66.47896575927734, + "logps/rejected": -750.052734375, + "loss": 1.073, + "margin_dpo/margin_mean": 275.4793395996094, + "margin_dpo/margin_std": 357.9488525390625, + "step": 424 + }, + { + "KL/chosen_KL_mean": -495.09136962890625, + "KL/mean": -594.1044921875, + "KL/rejected_KL_mean": -693.11767578125, + "KL/std": 313.45208740234375, + "epoch": 0.6240822320117474, + "fcm_dpo/beta": 0.001581181539222598, + "fcm_dpo/delta": 0.08807256817817688, + "fcm_dpo/margin": 198.02630615234375, + "fcm_dpo/q_t": 0.4265795946121216, + "grad_norm": 40.72890090942383, + "learning_rate": 1.8777780903377732e-07, + "logits/chosen": -0.6312949657440186, + "logits/rejected": -0.6337833404541016, + "logps/chosen": -541.9603271484375, + "logps/ref_chosen": -46.86899948120117, + "logps/ref_rejected": -95.92545318603516, + "logps/rejected": -789.0431518554688, + "loss": 1.1785, + "margin_dpo/margin_mean": 198.02630615234375, + "margin_dpo/margin_std": 365.4080505371094, + "step": 425 + }, + { + "KL/chosen_KL_mean": -447.80145263671875, + "KL/mean": -569.6341552734375, + "KL/rejected_KL_mean": -691.466796875, + "KL/std": 296.7207946777344, + "epoch": 0.6255506607929515, + "fcm_dpo/beta": 0.0015872271033003926, + "fcm_dpo/delta": 0.013328127562999725, + "fcm_dpo/margin": 243.6654052734375, + "fcm_dpo/q_t": 0.40938708186149597, + "grad_norm": 39.436275482177734, + "learning_rate": 1.8653568770724803e-07, + "logits/chosen": -0.6700231432914734, + "logits/rejected": -0.6314303874969482, + "logps/chosen": -524.385009765625, + "logps/ref_chosen": -76.58354187011719, + "logps/ref_rejected": -81.26658630371094, + "logps/rejected": -772.7333984375, + "loss": 1.0934, + "margin_dpo/margin_mean": 243.6654052734375, + "margin_dpo/margin_std": 301.1048583984375, + "step": 426 + }, + { + "KL/chosen_KL_mean": -404.7652587890625, + "KL/mean": -502.8582458496094, + "KL/rejected_KL_mean": -600.9512939453125, + "KL/std": 270.48968505859375, + "epoch": 0.6270190895741556, + "fcm_dpo/beta": 0.0015933769755065441, + "fcm_dpo/delta": 0.08883827924728394, + "fcm_dpo/margin": 196.18600463867188, + "fcm_dpo/q_t": 0.4279603958129883, + "grad_norm": 38.65625, + "learning_rate": 1.8529523872436977e-07, + "logits/chosen": -0.6598665714263916, + "logits/rejected": -0.6343759894371033, + "logps/chosen": -469.619140625, + "logps/ref_chosen": -64.8538818359375, + "logps/ref_rejected": -78.5660171508789, + "logps/rejected": -679.5172729492188, + "loss": 1.1568, + "margin_dpo/margin_mean": 196.18600463867188, + "margin_dpo/margin_std": 310.1150817871094, + "step": 427 + }, + { + "KL/chosen_KL_mean": -541.8001708984375, + "KL/mean": -685.3818359375, + "KL/rejected_KL_mean": -828.9635009765625, + "KL/std": 373.19793701171875, + "epoch": 0.6284875183553598, + "fcm_dpo/beta": 0.0015891847433522344, + "fcm_dpo/delta": -0.05788760259747505, + "fcm_dpo/margin": 287.1632080078125, + "fcm_dpo/q_t": 0.3983868360519409, + "grad_norm": 42.709285736083984, + "learning_rate": 1.8405649477212697e-07, + "logits/chosen": -0.6648718118667603, + "logits/rejected": -0.6649109125137329, + "logps/chosen": -604.4368896484375, + "logps/ref_chosen": -62.63666534423828, + "logps/ref_rejected": -103.28181457519531, + "logps/rejected": -932.2452392578125, + "loss": 1.0954, + "margin_dpo/margin_mean": 287.1632080078125, + "margin_dpo/margin_std": 445.24249267578125, + "step": 428 + }, + { + "KL/chosen_KL_mean": -558.6399536132812, + "KL/mean": -670.6189575195312, + "KL/rejected_KL_mean": -782.5979614257812, + "KL/std": 353.8562927246094, + "epoch": 0.6299559471365639, + "fcm_dpo/beta": 0.001599712180905044, + "fcm_dpo/delta": 0.04193819314241409, + "fcm_dpo/margin": 223.9580078125, + "fcm_dpo/q_t": 0.4191283583641052, + "grad_norm": 42.31879425048828, + "learning_rate": 1.828194884925749e-07, + "logits/chosen": -0.7320696711540222, + "logits/rejected": -0.7067664861679077, + "logps/chosen": -639.8739624023438, + "logps/ref_chosen": -81.23401641845703, + "logps/ref_rejected": -91.79493713378906, + "logps/rejected": -874.3928833007812, + "loss": 1.1686, + "margin_dpo/margin_mean": 223.9580078125, + "margin_dpo/margin_std": 418.2169494628906, + "step": 429 + }, + { + "KL/chosen_KL_mean": -485.84906005859375, + "KL/mean": -585.9974365234375, + "KL/rejected_KL_mean": -686.145751953125, + "KL/std": 333.52752685546875, + "epoch": 0.631424375917768, + "fcm_dpo/beta": 0.0016081253997981548, + "fcm_dpo/delta": 0.07896231859922409, + "fcm_dpo/margin": 200.2966766357422, + "fcm_dpo/q_t": 0.4268134832382202, + "grad_norm": 45.6950569152832, + "learning_rate": 1.8158425248197928e-07, + "logits/chosen": -0.6843982934951782, + "logits/rejected": -0.6812521815299988, + "logps/chosen": -546.7694091796875, + "logps/ref_chosen": -60.920326232910156, + "logps/ref_rejected": -104.42280578613281, + "logps/rejected": -790.5685424804688, + "loss": 1.1699, + "margin_dpo/margin_mean": 200.29666137695312, + "margin_dpo/margin_std": 362.1824035644531, + "step": 430 + }, + { + "KL/chosen_KL_mean": -476.85858154296875, + "KL/mean": -639.10986328125, + "KL/rejected_KL_mean": -801.361083984375, + "KL/std": 353.5206298828125, + "epoch": 0.6328928046989721, + "fcm_dpo/beta": 0.001593923894688487, + "fcm_dpo/delta": -0.12039424479007721, + "fcm_dpo/margin": 324.5025329589844, + "fcm_dpo/q_t": 0.3837839663028717, + "grad_norm": 37.95612716674805, + "learning_rate": 1.8035081928995788e-07, + "logits/chosen": -0.7060000896453857, + "logits/rejected": -0.7101027965545654, + "logps/chosen": -534.2073364257812, + "logps/ref_chosen": -57.34874725341797, + "logps/ref_rejected": -92.84022521972656, + "logps/rejected": -894.2013549804688, + "loss": 1.0314, + "margin_dpo/margin_mean": 324.5025329589844, + "margin_dpo/margin_std": 400.6416015625, + "step": 431 + }, + { + "KL/chosen_KL_mean": -433.40106201171875, + "KL/mean": -608.9974975585938, + "KL/rejected_KL_mean": -784.593994140625, + "KL/std": 369.924560546875, + "epoch": 0.6343612334801763, + "fcm_dpo/beta": 0.0015768797602504492, + "fcm_dpo/delta": -0.15814030170440674, + "fcm_dpo/margin": 351.19293212890625, + "fcm_dpo/q_t": 0.37685060501098633, + "grad_norm": 34.167293548583984, + "learning_rate": 1.791192214186223e-07, + "logits/chosen": -0.6776980757713318, + "logits/rejected": -0.6659233570098877, + "logps/chosen": -504.475830078125, + "logps/ref_chosen": -71.07479095458984, + "logps/ref_rejected": -98.57952880859375, + "logps/rejected": -883.1734619140625, + "loss": 0.9869, + "margin_dpo/margin_mean": 351.19293212890625, + "margin_dpo/margin_std": 373.4866943359375, + "step": 432 + }, + { + "KL/chosen_KL_mean": -559.2772827148438, + "KL/mean": -664.7277221679688, + "KL/rejected_KL_mean": -770.1781616210938, + "KL/std": 378.35809326171875, + "epoch": 0.6358296622613803, + "fcm_dpo/beta": 0.0015741533134132624, + "fcm_dpo/delta": 0.06906940788030624, + "fcm_dpo/margin": 210.90089416503906, + "fcm_dpo/q_t": 0.4204699993133545, + "grad_norm": 33.000389099121094, + "learning_rate": 1.7788949132172193e-07, + "logits/chosen": -0.7252877950668335, + "logits/rejected": -0.7109028697013855, + "logps/chosen": -617.5504760742188, + "logps/ref_chosen": -58.273193359375, + "logps/ref_rejected": -95.95089721679688, + "logps/rejected": -866.1290283203125, + "loss": 1.1957, + "margin_dpo/margin_mean": 210.90089416503906, + "margin_dpo/margin_std": 443.9112548828125, + "step": 433 + }, + { + "KL/chosen_KL_mean": -496.6466369628906, + "KL/mean": -609.65185546875, + "KL/rejected_KL_mean": -722.6571044921875, + "KL/std": 331.86968994140625, + "epoch": 0.6372980910425844, + "fcm_dpo/beta": 0.0015789458993822336, + "fcm_dpo/delta": 0.04392336681485176, + "fcm_dpo/margin": 226.01043701171875, + "fcm_dpo/q_t": 0.4241020679473877, + "grad_norm": 29.258861541748047, + "learning_rate": 1.7666166140378853e-07, + "logits/chosen": -0.7415832281112671, + "logits/rejected": -0.7311919927597046, + "logps/chosen": -558.620361328125, + "logps/ref_chosen": -61.97370147705078, + "logps/ref_rejected": -78.49861145019531, + "logps/rejected": -801.1557006835938, + "loss": 1.1536, + "margin_dpo/margin_mean": 226.01043701171875, + "margin_dpo/margin_std": 413.2788391113281, + "step": 434 + }, + { + "KL/chosen_KL_mean": -455.0494689941406, + "KL/mean": -597.1414184570312, + "KL/rejected_KL_mean": -739.2333374023438, + "KL/std": 330.17828369140625, + "epoch": 0.6387665198237885, + "fcm_dpo/beta": 0.0015741032548248768, + "fcm_dpo/delta": -0.048613518476486206, + "fcm_dpo/margin": 284.1838684082031, + "fcm_dpo/q_t": 0.39784517884254456, + "grad_norm": 33.527957916259766, + "learning_rate": 1.7543576401928218e-07, + "logits/chosen": -0.8012436628341675, + "logits/rejected": -0.784874677658081, + "logps/chosen": -506.551513671875, + "logps/ref_chosen": -51.502052307128906, + "logps/ref_rejected": -87.56689453125, + "logps/rejected": -826.8002319335938, + "loss": 1.065, + "margin_dpo/margin_mean": 284.183837890625, + "margin_dpo/margin_std": 358.4755859375, + "step": 435 + }, + { + "KL/chosen_KL_mean": -454.6264343261719, + "KL/mean": -576.6045532226562, + "KL/rejected_KL_mean": -698.5827026367188, + "KL/std": 299.61407470703125, + "epoch": 0.6402349486049926, + "fcm_dpo/beta": 0.0015738060465082526, + "fcm_dpo/delta": 0.016265233978629112, + "fcm_dpo/margin": 243.956298828125, + "fcm_dpo/q_t": 0.4141663908958435, + "grad_norm": 39.384552001953125, + "learning_rate": 1.742118314717391e-07, + "logits/chosen": -0.702154815196991, + "logits/rejected": -0.6535923480987549, + "logps/chosen": -526.0301513671875, + "logps/ref_chosen": -71.40371704101562, + "logps/ref_rejected": -82.72775268554688, + "logps/rejected": -781.3104248046875, + "loss": 1.1158, + "margin_dpo/margin_mean": 243.956298828125, + "margin_dpo/margin_std": 367.78271484375, + "step": 436 + }, + { + "KL/chosen_KL_mean": -440.7288513183594, + "KL/mean": -567.1207885742188, + "KL/rejected_KL_mean": -693.5126953125, + "KL/std": 293.3643798828125, + "epoch": 0.6417033773861968, + "fcm_dpo/beta": 0.0015777763910591602, + "fcm_dpo/delta": 0.0011859128717333078, + "fcm_dpo/margin": 252.7838897705078, + "fcm_dpo/q_t": 0.40734195709228516, + "grad_norm": 29.819622039794922, + "learning_rate": 1.7298989601292036e-07, + "logits/chosen": -0.7542532086372375, + "logits/rejected": -0.729715883731842, + "logps/chosen": -505.4731140136719, + "logps/ref_chosen": -64.7442626953125, + "logps/ref_rejected": -82.04356384277344, + "logps/rejected": -775.5562744140625, + "loss": 1.0905, + "margin_dpo/margin_mean": 252.78387451171875, + "margin_dpo/margin_std": 329.2221984863281, + "step": 437 + }, + { + "KL/chosen_KL_mean": -469.1627502441406, + "KL/mean": -613.673095703125, + "KL/rejected_KL_mean": -758.1834716796875, + "KL/std": 323.0591735839844, + "epoch": 0.6431718061674009, + "fcm_dpo/beta": 0.0015675770118832588, + "fcm_dpo/delta": -0.054624516516923904, + "fcm_dpo/margin": 289.0207214355469, + "fcm_dpo/q_t": 0.39459216594696045, + "grad_norm": 32.62807846069336, + "learning_rate": 1.7176998984196144e-07, + "logits/chosen": -0.7667056322097778, + "logits/rejected": -0.7360678911209106, + "logps/chosen": -528.181396484375, + "logps/ref_chosen": -59.0186653137207, + "logps/ref_rejected": -83.07682800292969, + "logps/rejected": -841.26025390625, + "loss": 1.05, + "margin_dpo/margin_mean": 289.0207214355469, + "margin_dpo/margin_std": 334.79864501953125, + "step": 438 + }, + { + "KL/chosen_KL_mean": -487.53643798828125, + "KL/mean": -603.5372314453125, + "KL/rejected_KL_mean": -719.5379638671875, + "KL/std": 336.40850830078125, + "epoch": 0.644640234948605, + "fcm_dpo/beta": 0.001555339666083455, + "fcm_dpo/delta": -0.08725128322839737, + "fcm_dpo/margin": 232.0015411376953, + "fcm_dpo/q_t": 0.41888073086738586, + "grad_norm": 40.83659744262695, + "learning_rate": 1.7055214510452458e-07, + "logits/chosen": -0.757964015007019, + "logits/rejected": -0.7572835683822632, + "logps/chosen": -541.3204956054688, + "logps/ref_chosen": -53.78407669067383, + "logps/ref_rejected": -83.98545837402344, + "logps/rejected": -803.5234375, + "loss": 1.1478, + "margin_dpo/margin_mean": 232.00155639648438, + "margin_dpo/margin_std": 380.4766845703125, + "step": 439 + }, + { + "KL/chosen_KL_mean": -491.5701904296875, + "KL/mean": -634.1220703125, + "KL/rejected_KL_mean": -776.6740112304688, + "KL/std": 404.11834716796875, + "epoch": 0.6461086637298091, + "fcm_dpo/beta": 0.0015516489511355758, + "fcm_dpo/delta": -0.043332286179065704, + "fcm_dpo/margin": 285.1037902832031, + "fcm_dpo/q_t": 0.4026166796684265, + "grad_norm": 44.514137268066406, + "learning_rate": 1.6933639389195134e-07, + "logits/chosen": -0.773739218711853, + "logits/rejected": -0.7669236660003662, + "logps/chosen": -570.1369018554688, + "logps/ref_chosen": -78.56671905517578, + "logps/ref_rejected": -96.49775695800781, + "logps/rejected": -873.1717529296875, + "loss": 1.0713, + "margin_dpo/margin_mean": 285.103759765625, + "margin_dpo/margin_std": 386.797119140625, + "step": 440 + }, + { + "KL/chosen_KL_mean": -591.2857666015625, + "KL/mean": -716.7557373046875, + "KL/rejected_KL_mean": -842.2255859375, + "KL/std": 407.85955810546875, + "epoch": 0.6475770925110133, + "fcm_dpo/beta": 0.001553539652377367, + "fcm_dpo/delta": 0.010096218436956406, + "fcm_dpo/margin": 250.93984985351562, + "fcm_dpo/q_t": 0.4177462160587311, + "grad_norm": 40.89066696166992, + "learning_rate": 1.681227682404166e-07, + "logits/chosen": -0.7674189805984497, + "logits/rejected": -0.7517750263214111, + "logps/chosen": -652.1102294921875, + "logps/ref_chosen": -60.824440002441406, + "logps/ref_rejected": -96.47080993652344, + "logps/rejected": -938.6964111328125, + "loss": 1.1743, + "margin_dpo/margin_mean": 250.93988037109375, + "margin_dpo/margin_std": 502.06768798828125, + "step": 441 + }, + { + "KL/chosen_KL_mean": -503.0433349609375, + "KL/mean": -659.7222900390625, + "KL/rejected_KL_mean": -816.4011840820312, + "KL/std": 401.5861511230469, + "epoch": 0.6490455212922174, + "fcm_dpo/beta": 0.0015460660215467215, + "fcm_dpo/delta": -0.08682595193386078, + "fcm_dpo/margin": 313.3577880859375, + "fcm_dpo/q_t": 0.3956022262573242, + "grad_norm": 36.8726806640625, + "learning_rate": 1.669113001300851e-07, + "logits/chosen": -0.7465405464172363, + "logits/rejected": -0.7388708591461182, + "logps/chosen": -550.0545654296875, + "logps/ref_chosen": -47.01121520996094, + "logps/ref_rejected": -76.53926086425781, + "logps/rejected": -892.9404296875, + "loss": 1.0721, + "margin_dpo/margin_mean": 313.3577880859375, + "margin_dpo/margin_std": 448.83795166015625, + "step": 442 + }, + { + "KL/chosen_KL_mean": -538.9011840820312, + "KL/mean": -627.173095703125, + "KL/rejected_KL_mean": -715.4451904296875, + "KL/std": 372.1048583984375, + "epoch": 0.6505139500734214, + "fcm_dpo/beta": 0.0015383637510240078, + "fcm_dpo/delta": 0.010316350497305393, + "fcm_dpo/margin": 176.54397583007812, + "fcm_dpo/q_t": 0.43916434049606323, + "grad_norm": 39.52730178833008, + "learning_rate": 1.6570202148426815e-07, + "logits/chosen": -0.733747124671936, + "logits/rejected": -0.710411548614502, + "logps/chosen": -610.1741943359375, + "logps/ref_chosen": -71.27301788330078, + "logps/ref_rejected": -86.679931640625, + "logps/rejected": -802.1251220703125, + "loss": 1.239, + "margin_dpo/margin_mean": 176.54397583007812, + "margin_dpo/margin_std": 431.0137634277344, + "step": 443 + }, + { + "KL/chosen_KL_mean": -545.4714965820312, + "KL/mean": -706.712890625, + "KL/rejected_KL_mean": -867.954345703125, + "KL/std": 410.585693359375, + "epoch": 0.6519823788546255, + "fcm_dpo/beta": 0.0015250996220856905, + "fcm_dpo/delta": -0.09439031034708023, + "fcm_dpo/margin": 322.4828796386719, + "fcm_dpo/q_t": 0.39275673031806946, + "grad_norm": 35.95709991455078, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": -0.6589124202728271, + "logits/rejected": -0.6668537259101868, + "logps/chosen": -602.6851806640625, + "logps/ref_chosen": -57.213706970214844, + "logps/ref_rejected": -97.25489807128906, + "logps/rejected": -965.209228515625, + "loss": 1.059, + "margin_dpo/margin_mean": 322.48291015625, + "margin_dpo/margin_std": 454.73382568359375, + "step": 444 + }, + { + "KL/chosen_KL_mean": -464.7125549316406, + "KL/mean": -615.153076171875, + "KL/rejected_KL_mean": -765.593505859375, + "KL/std": 321.0918273925781, + "epoch": 0.6534508076358296, + "fcm_dpo/beta": 0.0015192454447969794, + "fcm_dpo/delta": -0.058439984917640686, + "fcm_dpo/margin": 300.8808898925781, + "fcm_dpo/q_t": 0.39640486240386963, + "grad_norm": 32.19375228881836, + "learning_rate": 1.6329015999011182e-07, + "logits/chosen": -0.6859538555145264, + "logits/rejected": -0.6752911806106567, + "logps/chosen": -532.0123291015625, + "logps/ref_chosen": -67.29979705810547, + "logps/ref_rejected": -92.68267059326172, + "logps/rejected": -858.276123046875, + "loss": 1.0589, + "margin_dpo/margin_mean": 300.88092041015625, + "margin_dpo/margin_std": 378.2691650390625, + "step": 445 + }, + { + "KL/chosen_KL_mean": -440.65826416015625, + "KL/mean": -595.5625, + "KL/rejected_KL_mean": -750.4666748046875, + "KL/std": 347.827880859375, + "epoch": 0.6549192364170338, + "fcm_dpo/beta": 0.0015123614575713873, + "fcm_dpo/delta": -0.07043258845806122, + "fcm_dpo/margin": 309.8084716796875, + "fcm_dpo/q_t": 0.39075666666030884, + "grad_norm": 36.1616096496582, + "learning_rate": 1.6208764069656578e-07, + "logits/chosen": -0.7100153565406799, + "logits/rejected": -0.7190457582473755, + "logps/chosen": -499.7567138671875, + "logps/ref_chosen": -59.098487854003906, + "logps/ref_rejected": -101.26419067382812, + "logps/rejected": -851.7308349609375, + "loss": 1.0343, + "margin_dpo/margin_mean": 309.8084716796875, + "margin_dpo/margin_std": 329.4242248535156, + "step": 446 + }, + { + "KL/chosen_KL_mean": -485.10504150390625, + "KL/mean": -651.3726806640625, + "KL/rejected_KL_mean": -817.6403198242188, + "KL/std": 411.0166015625, + "epoch": 0.6563876651982379, + "fcm_dpo/beta": 0.0014915233477950096, + "fcm_dpo/delta": -0.09860781580209732, + "fcm_dpo/margin": 332.5352783203125, + "fcm_dpo/q_t": 0.39083898067474365, + "grad_norm": 33.39097595214844, + "learning_rate": 1.608874379754465e-07, + "logits/chosen": -0.7963074445724487, + "logits/rejected": -0.8120626211166382, + "logps/chosen": -541.1803588867188, + "logps/ref_chosen": -56.07533264160156, + "logps/ref_rejected": -98.69475555419922, + "logps/rejected": -916.3350830078125, + "loss": 1.0444, + "margin_dpo/margin_mean": 332.5352783203125, + "margin_dpo/margin_std": 443.71478271484375, + "step": 447 + }, + { + "KL/chosen_KL_mean": -549.9185791015625, + "KL/mean": -715.212646484375, + "KL/rejected_KL_mean": -880.5067138671875, + "KL/std": 387.88446044921875, + "epoch": 0.657856093979442, + "fcm_dpo/beta": 0.0014809533022344112, + "fcm_dpo/delta": -0.09178226441144943, + "fcm_dpo/margin": 330.588134765625, + "fcm_dpo/q_t": 0.3909049928188324, + "grad_norm": 37.11544418334961, + "learning_rate": 1.5968958345321177e-07, + "logits/chosen": -0.7098146677017212, + "logits/rejected": -0.7185966372489929, + "logps/chosen": -609.9224243164062, + "logps/ref_chosen": -60.00384521484375, + "logps/ref_rejected": -102.26465606689453, + "logps/rejected": -982.7713623046875, + "loss": 1.0476, + "margin_dpo/margin_mean": 330.588134765625, + "margin_dpo/margin_std": 435.3096923828125, + "step": 448 + }, + { + "KL/chosen_KL_mean": -565.3760986328125, + "KL/mean": -737.2021484375, + "KL/rejected_KL_mean": -909.028076171875, + "KL/std": 458.106689453125, + "epoch": 0.6593245227606461, + "fcm_dpo/beta": 0.001463989494368434, + "fcm_dpo/delta": -0.10579749941825867, + "fcm_dpo/margin": 343.6519470214844, + "fcm_dpo/q_t": 0.3931967616081238, + "grad_norm": 33.413673400878906, + "learning_rate": 1.584941086944423e-07, + "logits/chosen": -0.7576150894165039, + "logits/rejected": -0.746334433555603, + "logps/chosen": -632.9027099609375, + "logps/ref_chosen": -67.52661895751953, + "logps/ref_rejected": -88.59690856933594, + "logps/rejected": -997.625, + "loss": 1.0698, + "margin_dpo/margin_mean": 343.6519775390625, + "margin_dpo/margin_std": 530.8461303710938, + "step": 449 + }, + { + "KL/chosen_KL_mean": -459.6373596191406, + "KL/mean": -645.7894897460938, + "KL/rejected_KL_mean": -831.9415893554688, + "KL/std": 388.321044921875, + "epoch": 0.6607929515418502, + "fcm_dpo/beta": 0.0014468804001808167, + "fcm_dpo/delta": -0.14240342378616333, + "fcm_dpo/margin": 372.30426025390625, + "fcm_dpo/q_t": 0.37629586458206177, + "grad_norm": 43.87886428833008, + "learning_rate": 1.573010452010098e-07, + "logits/chosen": -0.8462213277816772, + "logits/rejected": -0.8595950603485107, + "logps/chosen": -516.7454833984375, + "logps/ref_chosen": -57.10811996459961, + "logps/ref_rejected": -102.75494384765625, + "logps/rejected": -934.696533203125, + "loss": 0.9856, + "margin_dpo/margin_mean": 372.30426025390625, + "margin_dpo/margin_std": 375.06005859375, + "step": 450 + }, + { + "KL/chosen_KL_mean": -655.2659912109375, + "KL/mean": -786.9049072265625, + "KL/rejected_KL_mean": -918.5439453125, + "KL/std": 500.51873779296875, + "epoch": 0.6622613803230544, + "fcm_dpo/beta": 0.0014441582607105374, + "fcm_dpo/delta": 0.02004837803542614, + "fcm_dpo/margin": 263.2779541015625, + "fcm_dpo/q_t": 0.4101364016532898, + "grad_norm": 50.99306106567383, + "learning_rate": 1.5611042441124687e-07, + "logits/chosen": -0.9174356460571289, + "logits/rejected": -0.8886551856994629, + "logps/chosen": -713.7348022460938, + "logps/ref_chosen": -58.46883010864258, + "logps/ref_rejected": -72.92941284179688, + "logps/rejected": -991.4733276367188, + "loss": 1.2051, + "margin_dpo/margin_mean": 263.2779541015625, + "margin_dpo/margin_std": 591.6530151367188, + "step": 451 + }, + { + "KL/chosen_KL_mean": -478.00341796875, + "KL/mean": -634.8177490234375, + "KL/rejected_KL_mean": -791.6319580078125, + "KL/std": 377.42657470703125, + "epoch": 0.6637298091042585, + "fcm_dpo/beta": 0.001435277285054326, + "fcm_dpo/delta": -0.05143965408205986, + "fcm_dpo/margin": 313.6285400390625, + "fcm_dpo/q_t": 0.3956111669540405, + "grad_norm": 30.187389373779297, + "learning_rate": 1.549222776991186e-07, + "logits/chosen": -0.8042168617248535, + "logits/rejected": -0.8225170373916626, + "logps/chosen": -528.3939819335938, + "logps/ref_chosen": -50.39055252075195, + "logps/ref_rejected": -97.77142333984375, + "logps/rejected": -889.4034423828125, + "loss": 1.0508, + "margin_dpo/margin_mean": 313.6285400390625, + "margin_dpo/margin_std": 363.00054931640625, + "step": 452 + }, + { + "KL/chosen_KL_mean": -546.2333984375, + "KL/mean": -708.6800537109375, + "KL/rejected_KL_mean": -871.1268310546875, + "KL/std": 392.1535949707031, + "epoch": 0.6651982378854625, + "fcm_dpo/beta": 0.0014257467119023204, + "fcm_dpo/delta": -0.06497231125831604, + "fcm_dpo/margin": 324.8934326171875, + "fcm_dpo/q_t": 0.3990872800350189, + "grad_norm": 34.372772216796875, + "learning_rate": 1.5373663637339584e-07, + "logits/chosen": -0.8440671563148499, + "logits/rejected": -0.8263705968856812, + "logps/chosen": -603.9482421875, + "logps/ref_chosen": -57.71485137939453, + "logps/ref_rejected": -82.20741271972656, + "logps/rejected": -953.334228515625, + "loss": 1.0778, + "margin_dpo/margin_mean": 324.8934631347656, + "margin_dpo/margin_std": 473.4591064453125, + "step": 453 + }, + { + "KL/chosen_KL_mean": -651.534912109375, + "KL/mean": -822.3595581054688, + "KL/rejected_KL_mean": -993.18408203125, + "KL/std": 444.50738525390625, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.0014141410356387496, + "fcm_dpo/delta": -0.08552803844213486, + "fcm_dpo/margin": 341.649169921875, + "fcm_dpo/q_t": 0.3941487669944763, + "grad_norm": 39.139339447021484, + "learning_rate": 1.5255353167683017e-07, + "logits/chosen": -0.8544209003448486, + "logits/rejected": -0.8400380611419678, + "logps/chosen": -712.4805908203125, + "logps/ref_chosen": -60.945648193359375, + "logps/ref_rejected": -84.95079040527344, + "logps/rejected": -1078.1348876953125, + "loss": 1.0725, + "margin_dpo/margin_mean": 341.649169921875, + "margin_dpo/margin_std": 507.084228515625, + "step": 454 + }, + { + "KL/chosen_KL_mean": -564.0460205078125, + "KL/mean": -756.771484375, + "KL/rejected_KL_mean": -949.4969482421875, + "KL/std": 434.53131103515625, + "epoch": 0.6681350954478708, + "fcm_dpo/beta": 0.0014031952014192939, + "fcm_dpo/delta": -0.14472892880439758, + "fcm_dpo/margin": 385.450927734375, + "fcm_dpo/q_t": 0.38020026683807373, + "grad_norm": 33.70637130737305, + "learning_rate": 1.5137299478533064e-07, + "logits/chosen": -0.8295320272445679, + "logits/rejected": -0.8505617380142212, + "logps/chosen": -608.9327392578125, + "logps/ref_chosen": -44.88671112060547, + "logps/ref_rejected": -115.30147552490234, + "logps/rejected": -1064.79833984375, + "loss": 1.0249, + "margin_dpo/margin_mean": 385.450927734375, + "margin_dpo/margin_std": 487.4344482421875, + "step": 455 + }, + { + "KL/chosen_KL_mean": -542.165771484375, + "KL/mean": -737.29248046875, + "KL/rejected_KL_mean": -932.4191284179688, + "KL/std": 445.7962341308594, + "epoch": 0.6696035242290749, + "fcm_dpo/beta": 0.0013802102766931057, + "fcm_dpo/delta": -0.14236783981323242, + "fcm_dpo/margin": 390.25335693359375, + "fcm_dpo/q_t": 0.37905341386795044, + "grad_norm": 34.32986831665039, + "learning_rate": 1.5019505680714232e-07, + "logits/chosen": -0.8049008846282959, + "logits/rejected": -0.829048752784729, + "logps/chosen": -599.2025756835938, + "logps/ref_chosen": -57.036781311035156, + "logps/ref_rejected": -105.21784210205078, + "logps/rejected": -1037.636962890625, + "loss": 0.9897, + "margin_dpo/margin_mean": 390.25335693359375, + "margin_dpo/margin_std": 416.21002197265625, + "step": 456 + }, + { + "KL/chosen_KL_mean": -505.49066162109375, + "KL/mean": -710.5162963867188, + "KL/rejected_KL_mean": -915.5419921875, + "KL/std": 429.17242431640625, + "epoch": 0.671071953010279, + "fcm_dpo/beta": 0.001355804386548698, + "fcm_dpo/delta": -0.1606726348400116, + "fcm_dpo/margin": 410.0513000488281, + "fcm_dpo/q_t": 0.37336650490760803, + "grad_norm": 50.008827209472656, + "learning_rate": 1.4901974878202627e-07, + "logits/chosen": -0.8348103761672974, + "logits/rejected": -0.8382035493850708, + "logps/chosen": -559.733154296875, + "logps/ref_chosen": -54.24253845214844, + "logps/ref_rejected": -85.10956573486328, + "logps/rejected": -1000.6514892578125, + "loss": 0.9782, + "margin_dpo/margin_mean": 410.0513000488281, + "margin_dpo/margin_std": 415.6295166015625, + "step": 457 + }, + { + "KL/chosen_KL_mean": -571.1600341796875, + "KL/mean": -721.54638671875, + "KL/rejected_KL_mean": -871.9327392578125, + "KL/std": 375.446044921875, + "epoch": 0.6725403817914831, + "fcm_dpo/beta": 0.0013467208482325077, + "fcm_dpo/delta": -0.005669653415679932, + "fcm_dpo/margin": 300.77264404296875, + "fcm_dpo/q_t": 0.4096975326538086, + "grad_norm": 33.263877868652344, + "learning_rate": 1.4784710168044212e-07, + "logits/chosen": -0.8518056273460388, + "logits/rejected": -0.840267539024353, + "logps/chosen": -626.56884765625, + "logps/ref_chosen": -55.40888214111328, + "logps/ref_rejected": -97.68325805664062, + "logps/rejected": -969.615966796875, + "loss": 1.0987, + "margin_dpo/margin_mean": 300.77264404296875, + "margin_dpo/margin_std": 419.0684814453125, + "step": 458 + }, + { + "KL/chosen_KL_mean": -596.946533203125, + "KL/mean": -784.0278930664062, + "KL/rejected_KL_mean": -971.109130859375, + "KL/std": 449.78326416015625, + "epoch": 0.6740088105726872, + "fcm_dpo/beta": 0.0013373238034546375, + "fcm_dpo/delta": -0.10349556058645248, + "fcm_dpo/margin": 374.16259765625, + "fcm_dpo/q_t": 0.3898807168006897, + "grad_norm": 37.392234802246094, + "learning_rate": 1.466771464027316e-07, + "logits/chosen": -0.8045568466186523, + "logits/rejected": -0.8199119567871094, + "logps/chosen": -643.5040283203125, + "logps/ref_chosen": -46.55748748779297, + "logps/ref_rejected": -86.16854095458984, + "logps/rejected": -1057.2777099609375, + "loss": 1.0568, + "margin_dpo/margin_mean": 374.16259765625, + "margin_dpo/margin_std": 516.4439086914062, + "step": 459 + }, + { + "KL/chosen_KL_mean": -642.6575317382812, + "KL/mean": -851.060791015625, + "KL/rejected_KL_mean": -1059.4639892578125, + "KL/std": 428.037109375, + "epoch": 0.6754772393538914, + "fcm_dpo/beta": 0.0013223644345998764, + "fcm_dpo/delta": -0.1554030030965805, + "fcm_dpo/margin": 416.8063049316406, + "fcm_dpo/q_t": 0.37675726413726807, + "grad_norm": 36.663360595703125, + "learning_rate": 1.4550991377830423e-07, + "logits/chosen": -0.8338453769683838, + "logits/rejected": -0.8649412393569946, + "logps/chosen": -694.29248046875, + "logps/ref_chosen": -51.63489532470703, + "logps/ref_rejected": -104.11935424804688, + "logps/rejected": -1163.583251953125, + "loss": 0.9875, + "margin_dpo/margin_mean": 416.80633544921875, + "margin_dpo/margin_std": 453.399658203125, + "step": 460 + }, + { + "KL/chosen_KL_mean": -657.5977172851562, + "KL/mean": -817.214599609375, + "KL/rejected_KL_mean": -976.83154296875, + "KL/std": 429.95391845703125, + "epoch": 0.6769456681350955, + "fcm_dpo/beta": 0.0013168307486921549, + "fcm_dpo/delta": -0.020892973989248276, + "fcm_dpo/margin": 319.23382568359375, + "fcm_dpo/q_t": 0.40674418210983276, + "grad_norm": 33.73992919921875, + "learning_rate": 1.4434543456482518e-07, + "logits/chosen": -0.858330249786377, + "logits/rejected": -0.8671220541000366, + "logps/chosen": -712.7796630859375, + "logps/ref_chosen": -55.18195724487305, + "logps/ref_rejected": -86.47689819335938, + "logps/rejected": -1063.3084716796875, + "loss": 1.0947, + "margin_dpo/margin_mean": 319.2338562011719, + "margin_dpo/margin_std": 468.2879333496094, + "step": 461 + }, + { + "KL/chosen_KL_mean": -621.3289794921875, + "KL/mean": -758.6487426757812, + "KL/rejected_KL_mean": -895.9684448242188, + "KL/std": 408.9393005371094, + "epoch": 0.6784140969162996, + "fcm_dpo/beta": 0.001316360547207296, + "fcm_dpo/delta": 0.03917883336544037, + "fcm_dpo/margin": 274.63946533203125, + "fcm_dpo/q_t": 0.4196895658969879, + "grad_norm": 41.01453399658203, + "learning_rate": 1.4318373944740484e-07, + "logits/chosen": -0.9267274141311646, + "logits/rejected": -0.9083724617958069, + "logps/chosen": -691.2569580078125, + "logps/ref_chosen": -69.92803192138672, + "logps/ref_rejected": -78.84111022949219, + "logps/rejected": -974.8095703125, + "loss": 1.1402, + "margin_dpo/margin_mean": 274.63946533203125, + "margin_dpo/margin_std": 454.64776611328125, + "step": 462 + }, + { + "KL/chosen_KL_mean": -614.42333984375, + "KL/mean": -786.9886474609375, + "KL/rejected_KL_mean": -959.553955078125, + "KL/std": 419.7926940917969, + "epoch": 0.6798825256975036, + "fcm_dpo/beta": 0.001315572764724493, + "fcm_dpo/delta": -0.05530533939599991, + "fcm_dpo/margin": 345.13055419921875, + "fcm_dpo/q_t": 0.39796507358551025, + "grad_norm": 32.41996765136719, + "learning_rate": 1.4202485903778976e-07, + "logits/chosen": -0.8691219687461853, + "logits/rejected": -0.8700894117355347, + "logps/chosen": -669.69775390625, + "logps/ref_chosen": -55.27437210083008, + "logps/ref_rejected": -89.02497863769531, + "logps/rejected": -1048.578857421875, + "loss": 1.0662, + "margin_dpo/margin_mean": 345.13055419921875, + "margin_dpo/margin_std": 457.62176513671875, + "step": 463 + }, + { + "KL/chosen_KL_mean": -600.5834350585938, + "KL/mean": -850.0731811523438, + "KL/rejected_KL_mean": -1099.56298828125, + "KL/std": 488.45526123046875, + "epoch": 0.6813509544787077, + "fcm_dpo/beta": 0.0012829885818064213, + "fcm_dpo/delta": -0.24955548346042633, + "fcm_dpo/margin": 498.9794921875, + "fcm_dpo/q_t": 0.3563224673271179, + "grad_norm": 42.92744827270508, + "learning_rate": 1.4086882387355658e-07, + "logits/chosen": -0.864883303642273, + "logits/rejected": -0.9228291511535645, + "logps/chosen": -651.4957275390625, + "logps/ref_chosen": -50.91230010986328, + "logps/ref_rejected": -102.4893798828125, + "logps/rejected": -1202.05224609375, + "loss": 0.9282, + "margin_dpo/margin_mean": 498.9794921875, + "margin_dpo/margin_std": 475.25860595703125, + "step": 464 + }, + { + "KL/chosen_KL_mean": -598.503662109375, + "KL/mean": -802.850341796875, + "KL/rejected_KL_mean": -1007.1970825195312, + "KL/std": 488.08880615234375, + "epoch": 0.6828193832599119, + "fcm_dpo/beta": 0.001269632950425148, + "fcm_dpo/delta": -0.12196629494428635, + "fcm_dpo/margin": 408.6933898925781, + "fcm_dpo/q_t": 0.37997591495513916, + "grad_norm": 30.996089935302734, + "learning_rate": 1.3971566441730714e-07, + "logits/chosen": -0.851067841053009, + "logits/rejected": -0.871573805809021, + "logps/chosen": -658.6204833984375, + "logps/ref_chosen": -60.116851806640625, + "logps/ref_rejected": -113.94602966308594, + "logps/rejected": -1121.14306640625, + "loss": 1.0264, + "margin_dpo/margin_mean": 408.6933898925781, + "margin_dpo/margin_std": 498.4493408203125, + "step": 465 + }, + { + "KL/chosen_KL_mean": -651.2394409179688, + "KL/mean": -829.18798828125, + "KL/rejected_KL_mean": -1007.1365966796875, + "KL/std": 461.8182067871094, + "epoch": 0.684287812041116, + "fcm_dpo/beta": 0.0012555132852867246, + "fcm_dpo/delta": -0.04844123125076294, + "fcm_dpo/margin": 355.89715576171875, + "fcm_dpo/q_t": 0.39813193678855896, + "grad_norm": 34.06509017944336, + "learning_rate": 1.3856541105586545e-07, + "logits/chosen": -0.8493870496749878, + "logits/rejected": -0.8454769849777222, + "logps/chosen": -704.1603393554688, + "logps/ref_chosen": -52.920921325683594, + "logps/ref_rejected": -90.3154296875, + "logps/rejected": -1097.4520263671875, + "loss": 1.074, + "margin_dpo/margin_mean": 355.89715576171875, + "margin_dpo/margin_std": 472.82452392578125, + "step": 466 + }, + { + "KL/chosen_KL_mean": -826.9212646484375, + "KL/mean": -1013.4310302734375, + "KL/rejected_KL_mean": -1199.940673828125, + "KL/std": 584.6832275390625, + "epoch": 0.6857562408223201, + "fcm_dpo/beta": 0.0012465352192521095, + "fcm_dpo/delta": -0.06744933128356934, + "fcm_dpo/margin": 373.0193786621094, + "fcm_dpo/q_t": 0.39987167716026306, + "grad_norm": 53.18272018432617, + "learning_rate": 1.3741809409947729e-07, + "logits/chosen": -0.9681617021560669, + "logits/rejected": -0.9566558599472046, + "logps/chosen": -905.6370849609375, + "logps/ref_chosen": -78.7158203125, + "logps/ref_rejected": -102.86019897460938, + "logps/rejected": -1302.8009033203125, + "loss": 1.1327, + "margin_dpo/margin_mean": 373.0194091796875, + "margin_dpo/margin_std": 667.3912963867188, + "step": 467 + }, + { + "KL/chosen_KL_mean": -637.194091796875, + "KL/mean": -866.535400390625, + "KL/rejected_KL_mean": -1095.876708984375, + "KL/std": 503.98370361328125, + "epoch": 0.6872246696035242, + "fcm_dpo/beta": 0.0012332710903137922, + "fcm_dpo/delta": -0.17055264115333557, + "fcm_dpo/margin": 458.6826477050781, + "fcm_dpo/q_t": 0.37581923604011536, + "grad_norm": 52.71803283691406, + "learning_rate": 1.362737437810114e-07, + "logits/chosen": -0.8924639225006104, + "logits/rejected": -0.9000778794288635, + "logps/chosen": -707.1294555664062, + "logps/ref_chosen": -69.93536376953125, + "logps/ref_rejected": -101.02880859375, + "logps/rejected": -1196.905517578125, + "loss": 0.9999, + "margin_dpo/margin_mean": 458.6826171875, + "margin_dpo/margin_std": 557.9136962890625, + "step": 468 + }, + { + "KL/chosen_KL_mean": -674.091796875, + "KL/mean": -917.2705078125, + "KL/rejected_KL_mean": -1160.44921875, + "KL/std": 503.095947265625, + "epoch": 0.6886930983847284, + "fcm_dpo/beta": 0.0012071155942976475, + "fcm_dpo/delta": -0.1936628669500351, + "fcm_dpo/margin": 486.35748291015625, + "fcm_dpo/q_t": 0.3694315552711487, + "grad_norm": 36.40598678588867, + "learning_rate": 1.351323902551631e-07, + "logits/chosen": -0.8922240734100342, + "logits/rejected": -0.9052385091781616, + "logps/chosen": -742.2164916992188, + "logps/ref_chosen": -68.12469482421875, + "logps/ref_rejected": -104.78640747070312, + "logps/rejected": -1265.235595703125, + "loss": 0.978, + "margin_dpo/margin_mean": 486.35748291015625, + "margin_dpo/margin_std": 537.0926513671875, + "step": 469 + }, + { + "KL/chosen_KL_mean": -584.605224609375, + "KL/mean": -782.6975708007812, + "KL/rejected_KL_mean": -980.7899169921875, + "KL/std": 488.2651672363281, + "epoch": 0.6901615271659325, + "fcm_dpo/beta": 0.0012003884185105562, + "fcm_dpo/delta": -0.07743942737579346, + "fcm_dpo/margin": 396.18463134765625, + "fcm_dpo/q_t": 0.392379492521286, + "grad_norm": 29.941707611083984, + "learning_rate": 1.339940635976592e-07, + "logits/chosen": -0.8477715253829956, + "logits/rejected": -0.8522796630859375, + "logps/chosen": -628.397216796875, + "logps/ref_chosen": -43.791927337646484, + "logps/ref_rejected": -82.70285034179688, + "logps/rejected": -1063.4927978515625, + "loss": 1.0593, + "margin_dpo/margin_mean": 396.18463134765625, + "margin_dpo/margin_std": 526.9808349609375, + "step": 470 + }, + { + "KL/chosen_KL_mean": -723.69287109375, + "KL/mean": -888.711669921875, + "KL/rejected_KL_mean": -1053.73046875, + "KL/std": 517.3129272460938, + "epoch": 0.6916299559471366, + "fcm_dpo/beta": 0.00119293003808707, + "fcm_dpo/delta": 0.006166655570268631, + "fcm_dpo/margin": 330.03759765625, + "fcm_dpo/q_t": 0.4141054153442383, + "grad_norm": 44.29683303833008, + "learning_rate": 1.3285879380446563e-07, + "logits/chosen": -0.9283441305160522, + "logits/rejected": -0.9297612905502319, + "logps/chosen": -787.0324096679688, + "logps/ref_chosen": -63.33952331542969, + "logps/ref_rejected": -83.61048126220703, + "logps/rejected": -1137.3409423828125, + "loss": 1.1301, + "margin_dpo/margin_mean": 330.0375671386719, + "margin_dpo/margin_std": 555.3355712890625, + "step": 471 + }, + { + "KL/chosen_KL_mean": -694.7491455078125, + "KL/mean": -900.8193359375, + "KL/rejected_KL_mean": -1106.8895263671875, + "KL/std": 589.9913940429688, + "epoch": 0.6930983847283406, + "fcm_dpo/beta": 0.001185831381008029, + "fcm_dpo/delta": -0.0912611111998558, + "fcm_dpo/margin": 412.1404113769531, + "fcm_dpo/q_t": 0.39623403549194336, + "grad_norm": 34.139041900634766, + "learning_rate": 1.317266107909975e-07, + "logits/chosen": -0.9295023679733276, + "logits/rejected": -0.9044417142868042, + "logps/chosen": -778.415283203125, + "logps/ref_chosen": -83.66610717773438, + "logps/ref_rejected": -117.20919799804688, + "logps/rejected": -1224.0987548828125, + "loss": 1.0802, + "margin_dpo/margin_mean": 412.140380859375, + "margin_dpo/margin_std": 622.5784912109375, + "step": 472 + }, + { + "KL/chosen_KL_mean": -844.9259033203125, + "KL/mean": -950.9039306640625, + "KL/rejected_KL_mean": -1056.8818359375, + "KL/std": 620.759521484375, + "epoch": 0.6945668135095447, + "fcm_dpo/beta": 0.0011900889221578836, + "fcm_dpo/delta": 0.04035104438662529, + "fcm_dpo/margin": 211.95591735839844, + "fcm_dpo/q_t": 0.44867947697639465, + "grad_norm": 131.445068359375, + "learning_rate": 1.3059754439133002e-07, + "logits/chosen": -0.8819540739059448, + "logits/rejected": -0.851367712020874, + "logps/chosen": -908.4229125976562, + "logps/ref_chosen": -63.49696731567383, + "logps/ref_rejected": -81.14657592773438, + "logps/rejected": -1138.0283203125, + "loss": 1.3351, + "margin_dpo/margin_mean": 211.9559326171875, + "margin_dpo/margin_std": 784.6436767578125, + "step": 473 + }, + { + "KL/chosen_KL_mean": -676.4264526367188, + "KL/mean": -852.287109375, + "KL/rejected_KL_mean": -1028.147705078125, + "KL/std": 525.454833984375, + "epoch": 0.6960352422907489, + "fcm_dpo/beta": 0.0011939804535359144, + "fcm_dpo/delta": -0.021219126880168915, + "fcm_dpo/margin": 351.7213439941406, + "fcm_dpo/q_t": 0.4053725600242615, + "grad_norm": 47.71774673461914, + "learning_rate": 1.2947162435741277e-07, + "logits/chosen": -0.8572415113449097, + "logits/rejected": -0.8637101650238037, + "logps/chosen": -729.0383911132812, + "logps/ref_chosen": -52.6119384765625, + "logps/ref_rejected": -90.08041381835938, + "logps/rejected": -1118.2281494140625, + "loss": 1.1248, + "margin_dpo/margin_mean": 351.7213439941406, + "margin_dpo/margin_std": 567.389892578125, + "step": 474 + }, + { + "KL/chosen_KL_mean": -492.31280517578125, + "KL/mean": -697.96044921875, + "KL/rejected_KL_mean": -903.608154296875, + "KL/std": 407.37347412109375, + "epoch": 0.697503671071953, + "fcm_dpo/beta": 0.0011786803370341659, + "fcm_dpo/delta": -0.0870213657617569, + "fcm_dpo/margin": 411.29534912109375, + "fcm_dpo/q_t": 0.38825133442878723, + "grad_norm": 56.91255569458008, + "learning_rate": 1.2834888035828596e-07, + "logits/chosen": -0.8645976781845093, + "logits/rejected": -0.8880435228347778, + "logps/chosen": -534.8079833984375, + "logps/ref_chosen": -42.49519348144531, + "logps/ref_rejected": -90.06294250488281, + "logps/rejected": -993.6710815429688, + "loss": 1.0216, + "margin_dpo/margin_mean": 411.29534912109375, + "margin_dpo/margin_std": 438.8066711425781, + "step": 475 + }, + { + "KL/chosen_KL_mean": -581.162109375, + "KL/mean": -763.0509033203125, + "KL/rejected_KL_mean": -944.939697265625, + "KL/std": 481.3084411621094, + "epoch": 0.6989720998531571, + "fcm_dpo/beta": 0.0011740537593141198, + "fcm_dpo/delta": -0.027722399681806564, + "fcm_dpo/margin": 363.7776184082031, + "fcm_dpo/q_t": 0.40231823921203613, + "grad_norm": 102.54533386230469, + "learning_rate": 1.2722934197929802e-07, + "logits/chosen": -0.8250603675842285, + "logits/rejected": -0.8420618772506714, + "logps/chosen": -624.1114501953125, + "logps/ref_chosen": -42.94938278198242, + "logps/ref_rejected": -73.71023559570312, + "logps/rejected": -1018.64990234375, + "loss": 1.0709, + "margin_dpo/margin_mean": 363.7776184082031, + "margin_dpo/margin_std": 458.6659240722656, + "step": 476 + }, + { + "KL/chosen_KL_mean": -576.864990234375, + "KL/mean": -748.666015625, + "KL/rejected_KL_mean": -920.467041015625, + "KL/std": 453.67510986328125, + "epoch": 0.7004405286343612, + "fcm_dpo/beta": 0.0011754365405067801, + "fcm_dpo/delta": -0.004048405215144157, + "fcm_dpo/margin": 343.6020812988281, + "fcm_dpo/q_t": 0.407484769821167, + "grad_norm": 50.94183349609375, + "learning_rate": 1.2611303872132631e-07, + "logits/chosen": -0.8757802248001099, + "logits/rejected": -0.8439153432846069, + "logps/chosen": -647.6376342773438, + "logps/ref_chosen": -70.77261352539062, + "logps/ref_rejected": -76.13737487792969, + "logps/rejected": -996.6044311523438, + "loss": 1.1156, + "margin_dpo/margin_mean": 343.6020812988281, + "margin_dpo/margin_std": 538.941650390625, + "step": 477 + }, + { + "KL/chosen_KL_mean": -512.1103515625, + "KL/mean": -712.9010009765625, + "KL/rejected_KL_mean": -913.691650390625, + "KL/std": 417.81170654296875, + "epoch": 0.7019089574155654, + "fcm_dpo/beta": 0.0011684978380799294, + "fcm_dpo/delta": -0.07088005542755127, + "fcm_dpo/margin": 401.581298828125, + "fcm_dpo/q_t": 0.39256751537323, + "grad_norm": 44.561859130859375, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": -0.7855898141860962, + "logits/rejected": -0.8061913251876831, + "logps/chosen": -553.5509033203125, + "logps/ref_chosen": -41.440513610839844, + "logps/ref_rejected": -85.36196899414062, + "logps/rejected": -999.0535888671875, + "loss": 1.0413, + "margin_dpo/margin_mean": 401.581298828125, + "margin_dpo/margin_std": 472.5474853515625, + "step": 478 + }, + { + "KL/chosen_KL_mean": -679.7722778320312, + "KL/mean": -867.8972778320312, + "KL/rejected_KL_mean": -1056.022216796875, + "KL/std": 525.4520874023438, + "epoch": 0.7033773861967695, + "fcm_dpo/beta": 0.0011651396052911878, + "fcm_dpo/delta": -0.03945866972208023, + "fcm_dpo/margin": 376.24993896484375, + "fcm_dpo/q_t": 0.4034544825553894, + "grad_norm": 36.7021484375, + "learning_rate": 1.2389025514492456e-07, + "logits/chosen": -0.8868274092674255, + "logits/rejected": -0.9125785827636719, + "logps/chosen": -733.68017578125, + "logps/ref_chosen": -53.907920837402344, + "logps/ref_rejected": -95.1163330078125, + "logps/rejected": -1151.1385498046875, + "loss": 1.1051, + "margin_dpo/margin_mean": 376.24993896484375, + "margin_dpo/margin_std": 603.150146484375, + "step": 479 + }, + { + "KL/chosen_KL_mean": -800.5087890625, + "KL/mean": -963.208740234375, + "KL/rejected_KL_mean": -1125.90869140625, + "KL/std": 507.62554931640625, + "epoch": 0.7048458149779736, + "fcm_dpo/beta": 0.0011562576983124018, + "fcm_dpo/delta": -0.07623150199651718, + "fcm_dpo/margin": 325.4000244140625, + "fcm_dpo/q_t": 0.4170445203781128, + "grad_norm": 49.098358154296875, + "learning_rate": 1.227838333989088e-07, + "logits/chosen": -0.9548571109771729, + "logits/rejected": -0.9503375291824341, + "logps/chosen": -859.1915283203125, + "logps/ref_chosen": -58.682701110839844, + "logps/ref_rejected": -82.93248748779297, + "logps/rejected": -1208.8411865234375, + "loss": 1.1562, + "margin_dpo/margin_mean": 325.4000244140625, + "margin_dpo/margin_std": 569.0355224609375, + "step": 480 + }, + { + "KL/chosen_KL_mean": -698.514404296875, + "KL/mean": -934.3134765625, + "KL/rejected_KL_mean": -1170.112548828125, + "KL/std": 530.9942016601562, + "epoch": 0.7063142437591777, + "fcm_dpo/beta": 0.001140213687904179, + "fcm_dpo/delta": -0.14147771894931793, + "fcm_dpo/margin": 471.5982360839844, + "fcm_dpo/q_t": 0.3806016147136688, + "grad_norm": 40.23090362548828, + "learning_rate": 1.2168076391719489e-07, + "logits/chosen": -0.994126558303833, + "logits/rejected": -1.0202418565750122, + "logps/chosen": -753.4786376953125, + "logps/ref_chosen": -54.964271545410156, + "logps/ref_rejected": -92.42044067382812, + "logps/rejected": -1262.532958984375, + "loss": 1.0124, + "margin_dpo/margin_mean": 471.5982666015625, + "margin_dpo/margin_std": 567.219482421875, + "step": 481 + }, + { + "KL/chosen_KL_mean": -790.0252685546875, + "KL/mean": -897.4144897460938, + "KL/rejected_KL_mean": -1004.8038330078125, + "KL/std": 556.745849609375, + "epoch": 0.7077826725403817, + "fcm_dpo/beta": 0.001139242434874177, + "fcm_dpo/delta": 0.06273117661476135, + "fcm_dpo/margin": 214.77850341796875, + "fcm_dpo/q_t": 0.4425661563873291, + "grad_norm": 51.694149017333984, + "learning_rate": 1.2058107576668938e-07, + "logits/chosen": -0.9190036058425903, + "logits/rejected": -0.90464186668396, + "logps/chosen": -857.5787353515625, + "logps/ref_chosen": -67.553466796875, + "logps/ref_rejected": -87.58953857421875, + "logps/rejected": -1092.393310546875, + "loss": 1.2784, + "margin_dpo/margin_mean": 214.7785186767578, + "margin_dpo/margin_std": 644.366943359375, + "step": 482 + }, + { + "KL/chosen_KL_mean": -724.80859375, + "KL/mean": -970.60693359375, + "KL/rejected_KL_mean": -1216.4053955078125, + "KL/std": 547.3455810546875, + "epoch": 0.7092511013215859, + "fcm_dpo/beta": 0.0011287643574178219, + "fcm_dpo/delta": -0.15940281748771667, + "fcm_dpo/margin": 491.59686279296875, + "fcm_dpo/q_t": 0.3793754577636719, + "grad_norm": 40.69145202636719, + "learning_rate": 1.194847979251979e-07, + "logits/chosen": -0.9518593549728394, + "logits/rejected": -0.9543057680130005, + "logps/chosen": -788.1383666992188, + "logps/ref_chosen": -63.32981872558594, + "logps/ref_rejected": -95.78697204589844, + "logps/rejected": -1312.1923828125, + "loss": 1.011, + "margin_dpo/margin_mean": 491.5968322753906, + "margin_dpo/margin_std": 612.5662841796875, + "step": 483 + }, + { + "KL/chosen_KL_mean": -609.849609375, + "KL/mean": -838.743408203125, + "KL/rejected_KL_mean": -1067.6370849609375, + "KL/std": 533.2965087890625, + "epoch": 0.71071953010279, + "fcm_dpo/beta": 0.0011179596185684204, + "fcm_dpo/delta": -0.1146901473402977, + "fcm_dpo/margin": 457.7874755859375, + "fcm_dpo/q_t": 0.3862311840057373, + "grad_norm": 33.894779205322266, + "learning_rate": 1.1839195928066101e-07, + "logits/chosen": -1.0216343402862549, + "logits/rejected": -1.041937232017517, + "logps/chosen": -668.98779296875, + "logps/ref_chosen": -59.13812255859375, + "logps/ref_rejected": -84.37144470214844, + "logps/rejected": -1152.008544921875, + "loss": 1.024, + "margin_dpo/margin_mean": 457.7874755859375, + "margin_dpo/margin_std": 545.956298828125, + "step": 484 + }, + { + "KL/chosen_KL_mean": -657.9251708984375, + "KL/mean": -873.3860473632812, + "KL/rejected_KL_mean": -1088.846923828125, + "KL/std": 551.51806640625, + "epoch": 0.7121879588839941, + "fcm_dpo/beta": 0.0011059865355491638, + "fcm_dpo/delta": -0.07843407988548279, + "fcm_dpo/margin": 430.9217529296875, + "fcm_dpo/q_t": 0.395374596118927, + "grad_norm": 30.792177200317383, + "learning_rate": 1.1730258863039347e-07, + "logits/chosen": -0.9116197824478149, + "logits/rejected": -0.9275870323181152, + "logps/chosen": -716.7747802734375, + "logps/ref_chosen": -58.849571228027344, + "logps/ref_rejected": -103.36408233642578, + "logps/rejected": -1192.2109375, + "loss": 1.0703, + "margin_dpo/margin_mean": 430.9217224121094, + "margin_dpo/margin_std": 620.1131591796875, + "step": 485 + }, + { + "KL/chosen_KL_mean": -730.796142578125, + "KL/mean": -973.44775390625, + "KL/rejected_KL_mean": -1216.099365234375, + "KL/std": 588.2183837890625, + "epoch": 0.7136563876651982, + "fcm_dpo/beta": 0.001091314246878028, + "fcm_dpo/delta": -0.1332990676164627, + "fcm_dpo/margin": 485.30328369140625, + "fcm_dpo/q_t": 0.38558250665664673, + "grad_norm": 42.85213851928711, + "learning_rate": 1.1621671468032493e-07, + "logits/chosen": -1.025226354598999, + "logits/rejected": -1.0336995124816895, + "logps/chosen": -786.0557861328125, + "logps/ref_chosen": -55.25966262817383, + "logps/ref_rejected": -92.13936614990234, + "logps/rejected": -1308.23876953125, + "loss": 1.0581, + "margin_dpo/margin_mean": 485.30328369140625, + "margin_dpo/margin_std": 714.6268310546875, + "step": 486 + }, + { + "KL/chosen_KL_mean": -738.9359130859375, + "KL/mean": -917.86376953125, + "KL/rejected_KL_mean": -1096.791748046875, + "KL/std": 529.3869018554688, + "epoch": 0.7151248164464024, + "fcm_dpo/beta": 0.0010912182042375207, + "fcm_dpo/delta": 0.009562283754348755, + "fcm_dpo/margin": 357.8558349609375, + "fcm_dpo/q_t": 0.40953782200813293, + "grad_norm": 35.427616119384766, + "learning_rate": 1.1513436604424378e-07, + "logits/chosen": -1.0364776849746704, + "logits/rejected": -1.0385382175445557, + "logps/chosen": -791.9991455078125, + "logps/ref_chosen": -53.06330871582031, + "logps/ref_rejected": -92.41883087158203, + "logps/rejected": -1189.2105712890625, + "loss": 1.1157, + "margin_dpo/margin_mean": 357.85589599609375, + "margin_dpo/margin_std": 545.7218017578125, + "step": 487 + }, + { + "KL/chosen_KL_mean": -610.1397705078125, + "KL/mean": -795.30224609375, + "KL/rejected_KL_mean": -980.4647216796875, + "KL/std": 488.50335693359375, + "epoch": 0.7165932452276065, + "fcm_dpo/beta": 0.0010911128483712673, + "fcm_dpo/delta": -0.00425250269472599, + "fcm_dpo/margin": 370.32489013671875, + "fcm_dpo/q_t": 0.4075971841812134, + "grad_norm": 37.004825592041016, + "learning_rate": 1.1405557124304335e-07, + "logits/chosen": -0.9542009830474854, + "logits/rejected": -0.9616404175758362, + "logps/chosen": -662.3679809570312, + "logps/ref_chosen": -52.22815704345703, + "logps/ref_rejected": -84.00656127929688, + "logps/rejected": -1064.4713134765625, + "loss": 1.0826, + "margin_dpo/margin_mean": 370.32489013671875, + "margin_dpo/margin_std": 467.35565185546875, + "step": 488 + }, + { + "KL/chosen_KL_mean": -572.46044921875, + "KL/mean": -741.5594482421875, + "KL/rejected_KL_mean": -910.658447265625, + "KL/std": 456.1171875, + "epoch": 0.7180616740088106, + "fcm_dpo/beta": 0.0010937793413177133, + "fcm_dpo/delta": 0.030445974320173264, + "fcm_dpo/margin": 338.1979675292969, + "fcm_dpo/q_t": 0.41659069061279297, + "grad_norm": 35.718448638916016, + "learning_rate": 1.1298035870396985e-07, + "logits/chosen": -0.9860827922821045, + "logits/rejected": -0.9753962755203247, + "logps/chosen": -628.4500732421875, + "logps/ref_chosen": -55.989627838134766, + "logps/ref_rejected": -79.39812469482422, + "logps/rejected": -990.0565185546875, + "loss": 1.1177, + "margin_dpo/margin_mean": 338.19793701171875, + "margin_dpo/margin_std": 492.15753173828125, + "step": 489 + }, + { + "KL/chosen_KL_mean": -665.18701171875, + "KL/mean": -841.7398071289062, + "KL/rejected_KL_mean": -1018.2926025390625, + "KL/std": 565.1947021484375, + "epoch": 0.7195301027900147, + "fcm_dpo/beta": 0.0010953254532068968, + "fcm_dpo/delta": 0.013375891372561455, + "fcm_dpo/margin": 353.1055908203125, + "fcm_dpo/q_t": 0.4144512116909027, + "grad_norm": 36.92960739135742, + "learning_rate": 1.1190875675987355e-07, + "logits/chosen": -0.9983633756637573, + "logits/rejected": -1.038588523864746, + "logps/chosen": -717.5534057617188, + "logps/ref_chosen": -52.36639404296875, + "logps/ref_rejected": -110.4090576171875, + "logps/rejected": -1128.70166015625, + "loss": 1.1514, + "margin_dpo/margin_mean": 353.1055603027344, + "margin_dpo/margin_std": 646.734130859375, + "step": 490 + }, + { + "KL/chosen_KL_mean": -616.6781616210938, + "KL/mean": -748.5814208984375, + "KL/rejected_KL_mean": -880.484619140625, + "KL/std": 459.8958740234375, + "epoch": 0.7209985315712188, + "fcm_dpo/beta": 0.0011040510144084692, + "fcm_dpo/delta": 0.11021701991558075, + "fcm_dpo/margin": 263.8064880371094, + "fcm_dpo/q_t": 0.43390926718711853, + "grad_norm": 33.199649810791016, + "learning_rate": 1.1084079364846241e-07, + "logits/chosen": -0.9711790084838867, + "logits/rejected": -0.9616298675537109, + "logps/chosen": -676.79443359375, + "logps/ref_chosen": -60.11626434326172, + "logps/ref_rejected": -73.27278900146484, + "logps/rejected": -953.7574462890625, + "loss": 1.1829, + "margin_dpo/margin_mean": 263.8064880371094, + "margin_dpo/margin_std": 483.6207580566406, + "step": 491 + }, + { + "KL/chosen_KL_mean": -635.3643798828125, + "KL/mean": -769.837890625, + "KL/rejected_KL_mean": -904.3115234375, + "KL/std": 484.90753173828125, + "epoch": 0.7224669603524229, + "fcm_dpo/beta": 0.0011131614446640015, + "fcm_dpo/delta": 0.10221239924430847, + "fcm_dpo/margin": 268.947021484375, + "fcm_dpo/q_t": 0.43284350633621216, + "grad_norm": 43.230899810791016, + "learning_rate": 1.097764975115576e-07, + "logits/chosen": -1.0389471054077148, + "logits/rejected": -1.0215075016021729, + "logps/chosen": -689.3585815429688, + "logps/ref_chosen": -53.994178771972656, + "logps/ref_rejected": -72.65962219238281, + "logps/rejected": -976.9710693359375, + "loss": 1.2, + "margin_dpo/margin_mean": 268.947021484375, + "margin_dpo/margin_std": 554.927978515625, + "step": 492 + }, + { + "KL/chosen_KL_mean": -699.6456298828125, + "KL/mean": -847.1690063476562, + "KL/rejected_KL_mean": -994.6923828125, + "KL/std": 572.62060546875, + "epoch": 0.723935389133627, + "fcm_dpo/beta": 0.0011133572552353144, + "fcm_dpo/delta": -0.03688516467809677, + "fcm_dpo/margin": 295.0467224121094, + "fcm_dpo/q_t": 0.42351529002189636, + "grad_norm": 34.844478607177734, + "learning_rate": 1.0871589639435203e-07, + "logits/chosen": -1.1079821586608887, + "logits/rejected": -1.0783543586730957, + "logps/chosen": -775.1428833007812, + "logps/ref_chosen": -75.49723815917969, + "logps/ref_rejected": -87.32301330566406, + "logps/rejected": -1082.015380859375, + "loss": 1.1584, + "margin_dpo/margin_mean": 295.04669189453125, + "margin_dpo/margin_std": 489.8016357421875, + "step": 493 + }, + { + "KL/chosen_KL_mean": -565.2142944335938, + "KL/mean": -797.517578125, + "KL/rejected_KL_mean": -1029.821044921875, + "KL/std": 503.0618896484375, + "epoch": 0.7254038179148311, + "fcm_dpo/beta": 0.0011037448421120644, + "fcm_dpo/delta": -0.11577919125556946, + "fcm_dpo/margin": 464.60662841796875, + "fcm_dpo/q_t": 0.38258957862854004, + "grad_norm": 38.89155960083008, + "learning_rate": 1.0765901824467166e-07, + "logits/chosen": -0.9311180114746094, + "logits/rejected": -0.9682860970497131, + "logps/chosen": -606.5736083984375, + "logps/ref_chosen": -41.35926818847656, + "logps/ref_rejected": -86.09136962890625, + "logps/rejected": -1115.912353515625, + "loss": 1.0082, + "margin_dpo/margin_mean": 464.606689453125, + "margin_dpo/margin_std": 502.99725341796875, + "step": 494 + }, + { + "KL/chosen_KL_mean": -633.6134033203125, + "KL/mean": -830.8568725585938, + "KL/rejected_KL_mean": -1028.1004638671875, + "KL/std": 498.0575866699219, + "epoch": 0.7268722466960352, + "fcm_dpo/beta": 0.0010981929954141378, + "fcm_dpo/delta": -0.033951401710510254, + "fcm_dpo/margin": 394.487060546875, + "fcm_dpo/q_t": 0.40414437651634216, + "grad_norm": 42.01950454711914, + "learning_rate": 1.0660589091223854e-07, + "logits/chosen": -1.0234172344207764, + "logits/rejected": -1.0299253463745117, + "logps/chosen": -697.1484375, + "logps/ref_chosen": -63.53507995605469, + "logps/ref_rejected": -91.42443084716797, + "logps/rejected": -1119.52490234375, + "loss": 1.0924, + "margin_dpo/margin_mean": 394.487060546875, + "margin_dpo/margin_std": 591.7906494140625, + "step": 495 + }, + { + "KL/chosen_KL_mean": -781.1953735351562, + "KL/mean": -882.2847900390625, + "KL/rejected_KL_mean": -983.3742065429688, + "KL/std": 428.71868896484375, + "epoch": 0.7283406754772394, + "fcm_dpo/beta": 0.0011013501789420843, + "fcm_dpo/delta": 0.08242875337600708, + "fcm_dpo/margin": 202.17886352539062, + "fcm_dpo/q_t": 0.44761157035827637, + "grad_norm": 73.75658416748047, + "learning_rate": 1.0555654214793722e-07, + "logits/chosen": -1.0234228372573853, + "logits/rejected": -0.9919195175170898, + "logps/chosen": -853.787353515625, + "logps/ref_chosen": -72.5919189453125, + "logps/ref_rejected": -84.32933807373047, + "logps/rejected": -1067.70361328125, + "loss": 1.2357, + "margin_dpo/margin_mean": 202.17886352539062, + "margin_dpo/margin_std": 450.7599792480469, + "step": 496 + }, + { + "KL/chosen_KL_mean": -759.360595703125, + "KL/mean": -857.68359375, + "KL/rejected_KL_mean": -956.0064697265625, + "KL/std": 512.5221557617188, + "epoch": 0.7298091042584435, + "fcm_dpo/beta": 0.0011091852793470025, + "fcm_dpo/delta": 0.029760103672742844, + "fcm_dpo/margin": 196.64585876464844, + "fcm_dpo/q_t": 0.4496675729751587, + "grad_norm": 32.76177978515625, + "learning_rate": 1.0451099960308374e-07, + "logits/chosen": -0.9546867609024048, + "logits/rejected": -0.935001015663147, + "logps/chosen": -817.95458984375, + "logps/ref_chosen": -58.59397506713867, + "logps/ref_rejected": -76.28836822509766, + "logps/rejected": -1032.294921875, + "loss": 1.2508, + "margin_dpo/margin_mean": 196.6458740234375, + "margin_dpo/margin_std": 478.19500732421875, + "step": 497 + }, + { + "KL/chosen_KL_mean": -722.9497680664062, + "KL/mean": -899.7124633789062, + "KL/rejected_KL_mean": -1076.47509765625, + "KL/std": 522.7940063476562, + "epoch": 0.7312775330396476, + "fcm_dpo/beta": 0.0011089190375059843, + "fcm_dpo/delta": 0.008106292225420475, + "fcm_dpo/margin": 353.52532958984375, + "fcm_dpo/q_t": 0.4107200503349304, + "grad_norm": 31.680055618286133, + "learning_rate": 1.0346929082869641e-07, + "logits/chosen": -0.9777836799621582, + "logits/rejected": -0.9642585515975952, + "logps/chosen": -794.1553955078125, + "logps/ref_chosen": -71.20565795898438, + "logps/ref_rejected": -83.95803833007812, + "logps/rejected": -1160.43310546875, + "loss": 1.1266, + "margin_dpo/margin_mean": 353.5253601074219, + "margin_dpo/margin_std": 576.6744384765625, + "step": 498 + }, + { + "KL/chosen_KL_mean": -631.90966796875, + "KL/mean": -870.5003051757812, + "KL/rejected_KL_mean": -1109.091064453125, + "KL/std": 562.7127685546875, + "epoch": 0.7327459618208517, + "fcm_dpo/beta": 0.0010997585486620665, + "fcm_dpo/delta": -0.1281210333108902, + "fcm_dpo/margin": 477.1813049316406, + "fcm_dpo/q_t": 0.38377606868743896, + "grad_norm": 32.53940963745117, + "learning_rate": 1.0243144327477013e-07, + "logits/chosen": -0.988601565361023, + "logits/rejected": -1.0279262065887451, + "logps/chosen": -683.1648559570312, + "logps/ref_chosen": -51.25519561767578, + "logps/ref_rejected": -101.07870483398438, + "logps/rejected": -1210.169677734375, + "loss": 1.0295, + "margin_dpo/margin_mean": 477.1813049316406, + "margin_dpo/margin_std": 616.3780517578125, + "step": 499 + }, + { + "KL/chosen_KL_mean": -733.3824462890625, + "KL/mean": -922.1387329101562, + "KL/rejected_KL_mean": -1110.89501953125, + "KL/std": 483.62493896484375, + "epoch": 0.7342143906020558, + "fcm_dpo/beta": 0.0010943380184471607, + "fcm_dpo/delta": -0.013418500311672688, + "fcm_dpo/margin": 377.5125427246094, + "fcm_dpo/q_t": 0.4065711498260498, + "grad_norm": 32.50249099731445, + "learning_rate": 1.0139748428955333e-07, + "logits/chosen": -0.9592075347900391, + "logits/rejected": -0.9868057370185852, + "logps/chosen": -790.409912109375, + "logps/ref_chosen": -57.027442932128906, + "logps/ref_rejected": -93.93421173095703, + "logps/rejected": -1204.8291015625, + "loss": 1.1156, + "margin_dpo/margin_mean": 377.5125732421875, + "margin_dpo/margin_std": 596.400634765625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -652.4368286132812, + "KL/mean": -851.3330688476562, + "KL/rejected_KL_mean": -1050.229248046875, + "KL/std": 501.94122314453125, + "epoch": 0.73568281938326, + "fcm_dpo/beta": 0.0010930509306490421, + "fcm_dpo/delta": -0.03560645133256912, + "fcm_dpo/margin": 397.7924499511719, + "fcm_dpo/q_t": 0.40338021516799927, + "grad_norm": 31.65900993347168, + "learning_rate": 1.0036744111882672e-07, + "logits/chosen": -0.9449106454849243, + "logits/rejected": -0.9296088218688965, + "logps/chosen": -706.79638671875, + "logps/ref_chosen": -54.359527587890625, + "logps/ref_rejected": -80.15670013427734, + "logps/rejected": -1130.385986328125, + "loss": 1.1051, + "margin_dpo/margin_mean": 397.79241943359375, + "margin_dpo/margin_std": 624.0454711914062, + "step": 501 + }, + { + "KL/chosen_KL_mean": -590.9495849609375, + "KL/mean": -798.0103149414062, + "KL/rejected_KL_mean": -1005.0711059570312, + "KL/std": 499.19720458984375, + "epoch": 0.737151248164464, + "fcm_dpo/beta": 0.0010861887130886316, + "fcm_dpo/delta": -0.050974659621715546, + "fcm_dpo/margin": 414.1214599609375, + "fcm_dpo/q_t": 0.401166707277298, + "grad_norm": 28.097309112548828, + "learning_rate": 9.934134090518592e-08, + "logits/chosen": -0.8737450838088989, + "logits/rejected": -0.8553862571716309, + "logps/chosen": -658.550048828125, + "logps/ref_chosen": -67.60050964355469, + "logps/ref_rejected": -82.94876098632812, + "logps/rejected": -1088.019775390625, + "loss": 1.0626, + "margin_dpo/margin_mean": 414.1214599609375, + "margin_dpo/margin_std": 551.1231689453125, + "step": 502 + }, + { + "KL/chosen_KL_mean": -591.0940551757812, + "KL/mean": -777.262939453125, + "KL/rejected_KL_mean": -963.431884765625, + "KL/std": 456.4488220214844, + "epoch": 0.7386196769456681, + "fcm_dpo/beta": 0.0010829106904566288, + "fcm_dpo/delta": -0.003399871289730072, + "fcm_dpo/margin": 372.33782958984375, + "fcm_dpo/q_t": 0.4094755947589874, + "grad_norm": 30.805957794189453, + "learning_rate": 9.831921068732571e-08, + "logits/chosen": -0.8509905338287354, + "logits/rejected": -0.8399511575698853, + "logps/chosen": -646.1724853515625, + "logps/ref_chosen": -55.078407287597656, + "logps/ref_rejected": -82.50544738769531, + "logps/rejected": -1045.937255859375, + "loss": 1.0941, + "margin_dpo/margin_mean": 372.33782958984375, + "margin_dpo/margin_std": 513.9088134765625, + "step": 503 + }, + { + "KL/chosen_KL_mean": -612.4645385742188, + "KL/mean": -844.0081176757812, + "KL/rejected_KL_mean": -1075.5516357421875, + "KL/std": 534.1448974609375, + "epoch": 0.7400881057268722, + "fcm_dpo/beta": 0.0010781993623822927, + "fcm_dpo/delta": -0.10178365558385849, + "fcm_dpo/margin": 463.0871887207031, + "fcm_dpo/q_t": 0.38906794786453247, + "grad_norm": 28.29525375366211, + "learning_rate": 9.730107739932805e-08, + "logits/chosen": -0.9321706295013428, + "logits/rejected": -0.9592180252075195, + "logps/chosen": -672.4302978515625, + "logps/ref_chosen": -59.96575164794922, + "logps/ref_rejected": -103.76212310791016, + "logps/rejected": -1179.313720703125, + "loss": 1.0388, + "margin_dpo/margin_mean": 463.087158203125, + "margin_dpo/margin_std": 572.2286987304688, + "step": 504 + }, + { + "KL/chosen_KL_mean": -688.0867919921875, + "KL/mean": -798.6611938476562, + "KL/rejected_KL_mean": -909.235595703125, + "KL/std": 483.4139099121094, + "epoch": 0.7415565345080763, + "fcm_dpo/beta": 0.0010871076956391335, + "fcm_dpo/delta": 0.1614731252193451, + "fcm_dpo/margin": 221.14871215820312, + "fcm_dpo/q_t": 0.44385814666748047, + "grad_norm": 40.906707763671875, + "learning_rate": 9.628696786995188e-08, + "logits/chosen": -0.9417963027954102, + "logits/rejected": -0.9116191267967224, + "logps/chosen": -764.24169921875, + "logps/ref_chosen": -76.1549072265625, + "logps/ref_rejected": -88.58537292480469, + "logps/rejected": -997.8209228515625, + "loss": 1.2195, + "margin_dpo/margin_mean": 221.14871215820312, + "margin_dpo/margin_std": 449.42291259765625, + "step": 505 + }, + { + "KL/chosen_KL_mean": -552.0079345703125, + "KL/mean": -739.86669921875, + "KL/rejected_KL_mean": -927.7255249023438, + "KL/std": 480.6181640625, + "epoch": 0.7430249632892805, + "fcm_dpo/beta": 0.001087966375052929, + "fcm_dpo/delta": -0.009178068488836288, + "fcm_dpo/margin": 375.71759033203125, + "fcm_dpo/q_t": 0.40685611963272095, + "grad_norm": 41.05369567871094, + "learning_rate": 9.527690882192635e-08, + "logits/chosen": -0.9096465110778809, + "logits/rejected": -0.9178716540336609, + "logps/chosen": -600.9683837890625, + "logps/ref_chosen": -48.96050262451172, + "logps/ref_rejected": -78.41505432128906, + "logps/rejected": -1006.1405639648438, + "loss": 1.0943, + "margin_dpo/margin_mean": 375.71759033203125, + "margin_dpo/margin_std": 518.1092529296875, + "step": 506 + }, + { + "KL/chosen_KL_mean": -654.116943359375, + "KL/mean": -816.158447265625, + "KL/rejected_KL_mean": -978.199951171875, + "KL/std": 556.29931640625, + "epoch": 0.7444933920704846, + "fcm_dpo/beta": 0.0010937817860394716, + "fcm_dpo/delta": 0.04635169357061386, + "fcm_dpo/margin": 324.0830383300781, + "fcm_dpo/q_t": 0.4228259325027466, + "grad_norm": 30.889352798461914, + "learning_rate": 9.427092687124691e-08, + "logits/chosen": -0.957282543182373, + "logits/rejected": -0.95253586769104, + "logps/chosen": -720.9183959960938, + "logps/ref_chosen": -66.80149841308594, + "logps/ref_rejected": -95.37289428710938, + "logps/rejected": -1073.5728759765625, + "loss": 1.1562, + "margin_dpo/margin_mean": 324.08306884765625, + "margin_dpo/margin_std": 591.4808959960938, + "step": 507 + }, + { + "KL/chosen_KL_mean": -661.949951171875, + "KL/mean": -807.3236083984375, + "KL/rejected_KL_mean": -952.697265625, + "KL/std": 519.9277954101562, + "epoch": 0.7459618208516887, + "fcm_dpo/beta": 0.0011030698660761118, + "fcm_dpo/delta": 0.08043861389160156, + "fcm_dpo/margin": 290.74737548828125, + "fcm_dpo/q_t": 0.4288504719734192, + "grad_norm": 35.57503890991211, + "learning_rate": 9.326904852647344e-08, + "logits/chosen": -0.9091905355453491, + "logits/rejected": -0.9017305374145508, + "logps/chosen": -733.25341796875, + "logps/ref_chosen": -71.303466796875, + "logps/ref_rejected": -95.6275405883789, + "logps/rejected": -1048.3248291015625, + "loss": 1.1934, + "margin_dpo/margin_mean": 290.74737548828125, + "margin_dpo/margin_std": 599.302734375, + "step": 508 + }, + { + "KL/chosen_KL_mean": -501.7856750488281, + "KL/mean": -661.9292602539062, + "KL/rejected_KL_mean": -822.0728149414062, + "KL/std": 382.68975830078125, + "epoch": 0.7474302496328928, + "fcm_dpo/beta": 0.0011099616531282663, + "fcm_dpo/delta": 0.04505918547511101, + "fcm_dpo/margin": 320.2871398925781, + "fcm_dpo/q_t": 0.4187580645084381, + "grad_norm": 29.257396697998047, + "learning_rate": 9.227130018803195e-08, + "logits/chosen": -0.814157247543335, + "logits/rejected": -0.7995736598968506, + "logps/chosen": -565.6046142578125, + "logps/ref_chosen": -63.81895065307617, + "logps/ref_rejected": -83.25643920898438, + "logps/rejected": -905.3292236328125, + "loss": 1.1258, + "margin_dpo/margin_mean": 320.287109375, + "margin_dpo/margin_std": 466.1398010253906, + "step": 509 + }, + { + "KL/chosen_KL_mean": -626.7471923828125, + "KL/mean": -820.874755859375, + "KL/rejected_KL_mean": -1015.002197265625, + "KL/std": 441.17962646484375, + "epoch": 0.748898678414097, + "fcm_dpo/beta": 0.0011074226349592209, + "fcm_dpo/delta": -0.030622189864516258, + "fcm_dpo/margin": 388.2550354003906, + "fcm_dpo/q_t": 0.3993905484676361, + "grad_norm": 29.603723526000977, + "learning_rate": 9.127770814751932e-08, + "logits/chosen": -0.8510131239891052, + "logits/rejected": -0.8649104237556458, + "logps/chosen": -678.6256103515625, + "logps/ref_chosen": -51.878448486328125, + "logps/ref_rejected": -102.7651596069336, + "logps/rejected": -1117.767333984375, + "loss": 1.0558, + "margin_dpo/margin_mean": 388.25506591796875, + "margin_dpo/margin_std": 431.11676025390625, + "step": 510 + }, + { + "KL/chosen_KL_mean": -593.8864135742188, + "KL/mean": -752.0758056640625, + "KL/rejected_KL_mean": -910.26513671875, + "KL/std": 475.2417907714844, + "epoch": 0.750367107195301, + "fcm_dpo/beta": 0.0011101202107965946, + "fcm_dpo/delta": 0.04963508993387222, + "fcm_dpo/margin": 316.3787841796875, + "fcm_dpo/q_t": 0.4193497896194458, + "grad_norm": 29.786012649536133, + "learning_rate": 9.028829858700973e-08, + "logits/chosen": -0.9209908246994019, + "logits/rejected": -0.9174632430076599, + "logps/chosen": -654.12451171875, + "logps/ref_chosen": -60.23811721801758, + "logps/ref_rejected": -92.85676574707031, + "logps/rejected": -1003.1219482421875, + "loss": 1.1544, + "margin_dpo/margin_mean": 316.3787536621094, + "margin_dpo/margin_std": 560.738037109375, + "step": 511 + }, + { + "KL/chosen_KL_mean": -489.36114501953125, + "KL/mean": -704.290283203125, + "KL/rejected_KL_mean": -919.219482421875, + "KL/std": 439.2633056640625, + "epoch": 0.7518355359765051, + "fcm_dpo/beta": 0.001103471266105771, + "fcm_dpo/delta": -0.07630272209644318, + "fcm_dpo/margin": 429.8582763671875, + "fcm_dpo/q_t": 0.3897845149040222, + "grad_norm": 37.09547805786133, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": -0.8986982107162476, + "logits/rejected": -0.9108966588973999, + "logps/chosen": -544.2666625976562, + "logps/ref_chosen": -54.905494689941406, + "logps/ref_rejected": -81.87586975097656, + "logps/rejected": -1001.0953369140625, + "loss": 1.0214, + "margin_dpo/margin_mean": 429.8583068847656, + "margin_dpo/margin_std": 438.10321044921875, + "step": 512 + }, + { + "KL/chosen_KL_mean": -636.2532958984375, + "KL/mean": -780.3316650390625, + "KL/rejected_KL_mean": -924.4100341796875, + "KL/std": 423.2544250488281, + "epoch": 0.7533039647577092, + "fcm_dpo/beta": 0.00109992700163275, + "fcm_dpo/delta": -0.025476014241576195, + "fcm_dpo/margin": 288.15679931640625, + "fcm_dpo/q_t": 0.4271167516708374, + "grad_norm": 44.68075942993164, + "learning_rate": 8.832213108254863e-08, + "logits/chosen": -0.9700770378112793, + "logits/rejected": -0.9396867752075195, + "logps/chosen": -701.169677734375, + "logps/ref_chosen": -64.91644287109375, + "logps/ref_rejected": -76.06245422363281, + "logps/rejected": -1000.4725341796875, + "loss": 1.1665, + "margin_dpo/margin_mean": 288.15679931640625, + "margin_dpo/margin_std": 489.13134765625, + "step": 513 + }, + { + "KL/chosen_KL_mean": -640.1827392578125, + "KL/mean": -787.67919921875, + "KL/rejected_KL_mean": -935.1756591796875, + "KL/std": 454.5134582519531, + "epoch": 0.7547723935389133, + "fcm_dpo/beta": 0.00110641831997782, + "fcm_dpo/delta": 0.07481080293655396, + "fcm_dpo/margin": 294.99298095703125, + "fcm_dpo/q_t": 0.42692023515701294, + "grad_norm": 36.836143493652344, + "learning_rate": 8.734542494893954e-08, + "logits/chosen": -0.9246986508369446, + "logits/rejected": -0.904691755771637, + "logps/chosen": -714.412353515625, + "logps/ref_chosen": -74.22957611083984, + "logps/ref_rejected": -78.945556640625, + "logps/rejected": -1014.1212158203125, + "loss": 1.1608, + "margin_dpo/margin_mean": 294.99298095703125, + "margin_dpo/margin_std": 514.4725341796875, + "step": 514 + }, + { + "KL/chosen_KL_mean": -525.1871337890625, + "KL/mean": -651.1346435546875, + "KL/rejected_KL_mean": -777.08203125, + "KL/std": 396.79437255859375, + "epoch": 0.7562408223201175, + "fcm_dpo/beta": 0.0011181033914908767, + "fcm_dpo/delta": 0.12005515396595001, + "fcm_dpo/margin": 251.8948974609375, + "fcm_dpo/q_t": 0.4334961175918579, + "grad_norm": 34.989688873291016, + "learning_rate": 8.637300491465272e-08, + "logits/chosen": -0.8561180830001831, + "logits/rejected": -0.8611321449279785, + "logps/chosen": -575.5887451171875, + "logps/ref_chosen": -50.40156555175781, + "logps/ref_rejected": -87.09774780273438, + "logps/rejected": -864.1798095703125, + "loss": 1.1867, + "margin_dpo/margin_mean": 251.89492797851562, + "margin_dpo/margin_std": 460.27935791015625, + "step": 515 + }, + { + "KL/chosen_KL_mean": -596.61669921875, + "KL/mean": -770.6539306640625, + "KL/rejected_KL_mean": -944.6910400390625, + "KL/std": 452.91241455078125, + "epoch": 0.7577092511013216, + "fcm_dpo/beta": 0.0011240649037063122, + "fcm_dpo/delta": 0.00880388356745243, + "fcm_dpo/margin": 348.07440185546875, + "fcm_dpo/q_t": 0.408348023891449, + "grad_norm": 32.15882110595703, + "learning_rate": 8.540489660386064e-08, + "logits/chosen": -0.9794058799743652, + "logits/rejected": -0.9973136186599731, + "logps/chosen": -661.2662353515625, + "logps/ref_chosen": -64.64956665039062, + "logps/ref_rejected": -111.72237396240234, + "logps/rejected": -1056.4134521484375, + "loss": 1.0971, + "margin_dpo/margin_mean": 348.0743713378906, + "margin_dpo/margin_std": 459.40338134765625, + "step": 516 + }, + { + "KL/chosen_KL_mean": -583.4486083984375, + "KL/mean": -800.865478515625, + "KL/rejected_KL_mean": -1018.2823486328125, + "KL/std": 497.50537109375, + "epoch": 0.7591776798825257, + "fcm_dpo/beta": 0.0011140817077830434, + "fcm_dpo/delta": -0.0866817757487297, + "fcm_dpo/margin": 434.8336486816406, + "fcm_dpo/q_t": 0.39268559217453003, + "grad_norm": 31.913042068481445, + "learning_rate": 8.444112552711752e-08, + "logits/chosen": -0.93073570728302, + "logits/rejected": -0.9230749607086182, + "logps/chosen": -644.3621826171875, + "logps/ref_chosen": -60.913551330566406, + "logps/ref_rejected": -89.08308410644531, + "logps/rejected": -1107.365478515625, + "loss": 1.0456, + "margin_dpo/margin_mean": 434.83367919921875, + "margin_dpo/margin_std": 564.2802124023438, + "step": 517 + }, + { + "KL/chosen_KL_mean": -531.2779541015625, + "KL/mean": -708.3522338867188, + "KL/rejected_KL_mean": -885.426513671875, + "KL/std": 404.1721496582031, + "epoch": 0.7606461086637298, + "fcm_dpo/beta": 0.0011119421105831861, + "fcm_dpo/delta": 0.006271369755268097, + "fcm_dpo/margin": 354.1485900878906, + "fcm_dpo/q_t": 0.40764957666397095, + "grad_norm": 31.667274475097656, + "learning_rate": 8.348171708068747e-08, + "logits/chosen": -0.9522829055786133, + "logits/rejected": -0.9628884792327881, + "logps/chosen": -588.7338256835938, + "logps/ref_chosen": -57.45589065551758, + "logps/ref_rejected": -85.31269836425781, + "logps/rejected": -970.7392578125, + "loss": 1.0874, + "margin_dpo/margin_mean": 354.1485900878906, + "margin_dpo/margin_std": 437.1719970703125, + "step": 518 + }, + { + "KL/chosen_KL_mean": -544.94873046875, + "KL/mean": -669.02685546875, + "KL/rejected_KL_mean": -793.10498046875, + "KL/std": 379.0975646972656, + "epoch": 0.762114537444934, + "fcm_dpo/beta": 0.0011145803146064281, + "fcm_dpo/delta": 0.018153877928853035, + "fcm_dpo/margin": 248.15621948242188, + "fcm_dpo/q_t": 0.4351785182952881, + "grad_norm": 26.59686851501465, + "learning_rate": 8.25266965458755e-08, + "logits/chosen": -0.9192925691604614, + "logits/rejected": -0.8974975347518921, + "logps/chosen": -619.0120849609375, + "logps/ref_chosen": -74.06331634521484, + "logps/ref_rejected": -104.44416809082031, + "logps/rejected": -897.5491943359375, + "loss": 1.192, + "margin_dpo/margin_mean": 248.15621948242188, + "margin_dpo/margin_std": 452.4583435058594, + "step": 519 + }, + { + "KL/chosen_KL_mean": -566.6231689453125, + "KL/mean": -728.2750854492188, + "KL/rejected_KL_mean": -889.9268798828125, + "KL/std": 431.6119689941406, + "epoch": 0.7635829662261381, + "fcm_dpo/beta": 0.0011171854566782713, + "fcm_dpo/delta": 0.039489611983299255, + "fcm_dpo/margin": 323.3037109375, + "fcm_dpo/q_t": 0.41814154386520386, + "grad_norm": 34.39997100830078, + "learning_rate": 8.15760890883607e-08, + "logits/chosen": -0.8914647102355957, + "logits/rejected": -0.8902548551559448, + "logps/chosen": -636.9230346679688, + "logps/ref_chosen": -70.2998275756836, + "logps/ref_rejected": -99.98133850097656, + "logps/rejected": -989.9082641601562, + "loss": 1.1248, + "margin_dpo/margin_mean": 323.3037109375, + "margin_dpo/margin_std": 469.2210693359375, + "step": 520 + }, + { + "KL/chosen_KL_mean": -525.2251586914062, + "KL/mean": -698.4849243164062, + "KL/rejected_KL_mean": -871.74462890625, + "KL/std": 449.91253662109375, + "epoch": 0.7650513950073421, + "fcm_dpo/beta": 0.0011244425550103188, + "fcm_dpo/delta": 0.010120447725057602, + "fcm_dpo/margin": 346.51953125, + "fcm_dpo/q_t": 0.4106011986732483, + "grad_norm": 25.767515182495117, + "learning_rate": 8.062991975753378e-08, + "logits/chosen": -0.9617199897766113, + "logits/rejected": -0.9598067402839661, + "logps/chosen": -583.3680419921875, + "logps/ref_chosen": -58.14292526245117, + "logps/ref_rejected": -83.28060913085938, + "logps/rejected": -955.0252685546875, + "loss": 1.1012, + "margin_dpo/margin_mean": 346.51953125, + "margin_dpo/margin_std": 464.26416015625, + "step": 521 + }, + { + "KL/chosen_KL_mean": -578.2764892578125, + "KL/mean": -737.2076416015625, + "KL/rejected_KL_mean": -896.1387939453125, + "KL/std": 469.0858154296875, + "epoch": 0.7665198237885462, + "fcm_dpo/beta": 0.001124875620007515, + "fcm_dpo/delta": 0.04321271926164627, + "fcm_dpo/margin": 317.8622131347656, + "fcm_dpo/q_t": 0.4179668128490448, + "grad_norm": 27.95970916748047, + "learning_rate": 7.968821348583643e-08, + "logits/chosen": -0.9487807750701904, + "logits/rejected": -0.9443739652633667, + "logps/chosen": -624.82421875, + "logps/ref_chosen": -46.54766845703125, + "logps/ref_rejected": -66.01388549804688, + "logps/rejected": -962.1526489257812, + "loss": 1.1318, + "margin_dpo/margin_mean": 317.86224365234375, + "margin_dpo/margin_std": 489.20440673828125, + "step": 522 + }, + { + "KL/chosen_KL_mean": -616.9910888671875, + "KL/mean": -790.3553466796875, + "KL/rejected_KL_mean": -963.7196044921875, + "KL/std": 531.9453125, + "epoch": 0.7679882525697503, + "fcm_dpo/beta": 0.0011264176573604345, + "fcm_dpo/delta": 0.00962541438639164, + "fcm_dpo/margin": 346.72845458984375, + "fcm_dpo/q_t": 0.4115051031112671, + "grad_norm": 49.35715103149414, + "learning_rate": 7.875099508810484e-08, + "logits/chosen": -0.9954909086227417, + "logits/rejected": -0.9879505634307861, + "logps/chosen": -678.7607421875, + "logps/ref_chosen": -61.76960372924805, + "logps/ref_rejected": -83.76141357421875, + "logps/rejected": -1047.48095703125, + "loss": 1.1189, + "margin_dpo/margin_mean": 346.72845458984375, + "margin_dpo/margin_std": 540.300537109375, + "step": 523 + }, + { + "KL/chosen_KL_mean": -613.022705078125, + "KL/mean": -792.3504638671875, + "KL/rejected_KL_mean": -971.6781005859375, + "KL/std": 508.4144287109375, + "epoch": 0.7694566813509545, + "fcm_dpo/beta": 0.001124032773077488, + "fcm_dpo/delta": -0.00345630943775177, + "fcm_dpo/margin": 358.6553649902344, + "fcm_dpo/q_t": 0.4057971239089966, + "grad_norm": 34.90718078613281, + "learning_rate": 7.781828926091535e-08, + "logits/chosen": -1.0312086343765259, + "logits/rejected": -1.0126185417175293, + "logps/chosen": -691.0947875976562, + "logps/ref_chosen": -78.0720443725586, + "logps/ref_rejected": -81.30198669433594, + "logps/rejected": -1052.9801025390625, + "loss": 1.1073, + "margin_dpo/margin_mean": 358.6553649902344, + "margin_dpo/margin_std": 517.4310302734375, + "step": 524 + }, + { + "KL/chosen_KL_mean": -596.0318603515625, + "KL/mean": -821.548828125, + "KL/rejected_KL_mean": -1047.0657958984375, + "KL/std": 515.7371826171875, + "epoch": 0.7709251101321586, + "fcm_dpo/beta": 0.0011153742671012878, + "fcm_dpo/delta": -0.10615138709545135, + "fcm_dpo/margin": 451.0339660644531, + "fcm_dpo/q_t": 0.38702392578125, + "grad_norm": 34.25413131713867, + "learning_rate": 7.689012058193384e-08, + "logits/chosen": -0.9680420160293579, + "logits/rejected": -1.0015491247177124, + "logps/chosen": -646.8596801757812, + "logps/ref_chosen": -50.827857971191406, + "logps/ref_rejected": -100.05294036865234, + "logps/rejected": -1147.11865234375, + "loss": 1.026, + "margin_dpo/margin_mean": 451.0339660644531, + "margin_dpo/margin_std": 533.57666015625, + "step": 525 + }, + { + "KL/chosen_KL_mean": -641.201904296875, + "KL/mean": -872.0587158203125, + "KL/rejected_KL_mean": -1102.91552734375, + "KL/std": 532.249267578125, + "epoch": 0.7723935389133627, + "fcm_dpo/beta": 0.0011071430053561926, + "fcm_dpo/delta": -0.1140209287405014, + "fcm_dpo/margin": 461.7136535644531, + "fcm_dpo/q_t": 0.383984237909317, + "grad_norm": 31.422950744628906, + "learning_rate": 7.596651350926836e-08, + "logits/chosen": -0.9887993335723877, + "logits/rejected": -0.9786882400512695, + "logps/chosen": -704.369140625, + "logps/ref_chosen": -63.167236328125, + "logps/ref_rejected": -86.30934143066406, + "logps/rejected": -1189.224853515625, + "loss": 1.0411, + "margin_dpo/margin_mean": 461.713623046875, + "margin_dpo/margin_std": 597.789794921875, + "step": 526 + }, + { + "KL/chosen_KL_mean": -675.8800048828125, + "KL/mean": -825.571533203125, + "KL/rejected_KL_mean": -975.262939453125, + "KL/std": 547.1878662109375, + "epoch": 0.7738619676945668, + "fcm_dpo/beta": 0.0011063042329624295, + "fcm_dpo/delta": 0.06996496021747589, + "fcm_dpo/margin": 299.3829650878906, + "fcm_dpo/q_t": 0.4228494465351105, + "grad_norm": 37.20889663696289, + "learning_rate": 7.504749238082414e-08, + "logits/chosen": -1.1690475940704346, + "logits/rejected": -1.126373529434204, + "logps/chosen": -747.0086669921875, + "logps/ref_chosen": -71.12867736816406, + "logps/ref_rejected": -78.3425521850586, + "logps/rejected": -1053.60546875, + "loss": 1.1431, + "margin_dpo/margin_mean": 299.3829650878906, + "margin_dpo/margin_std": 453.30816650390625, + "step": 527 + }, + { + "KL/chosen_KL_mean": -692.7294921875, + "KL/mean": -880.4638671875, + "KL/rejected_KL_mean": -1068.1982421875, + "KL/std": 529.0624389648438, + "epoch": 0.775330396475771, + "fcm_dpo/beta": 0.0011081206612288952, + "fcm_dpo/delta": -0.016417687758803368, + "fcm_dpo/margin": 375.46881103515625, + "fcm_dpo/q_t": 0.4085617959499359, + "grad_norm": 37.37120056152344, + "learning_rate": 7.413308141366254e-08, + "logits/chosen": -1.0335596799850464, + "logits/rejected": -1.0141860246658325, + "logps/chosen": -760.8189697265625, + "logps/ref_chosen": -68.0894546508789, + "logps/ref_rejected": -93.91006469726562, + "logps/rejected": -1162.1083984375, + "loss": 1.1177, + "margin_dpo/margin_mean": 375.46881103515625, + "margin_dpo/margin_std": 612.314697265625, + "step": 528 + }, + { + "KL/chosen_KL_mean": -838.7152099609375, + "KL/mean": -968.595458984375, + "KL/rejected_KL_mean": -1098.4757080078125, + "KL/std": 483.984130859375, + "epoch": 0.7767988252569751, + "fcm_dpo/beta": 0.001110037206672132, + "fcm_dpo/delta": 0.017718147486448288, + "fcm_dpo/margin": 259.760498046875, + "fcm_dpo/q_t": 0.4340352416038513, + "grad_norm": 38.99552536010742, + "learning_rate": 7.322330470336313e-08, + "logits/chosen": -1.1005570888519287, + "logits/rejected": -1.1074516773223877, + "logps/chosen": -894.2901611328125, + "logps/ref_chosen": -55.57495880126953, + "logps/ref_rejected": -89.20909118652344, + "logps/rejected": -1187.684814453125, + "loss": 1.2326, + "margin_dpo/margin_mean": 259.760498046875, + "margin_dpo/margin_std": 621.6509399414062, + "step": 529 + }, + { + "KL/chosen_KL_mean": -708.62939453125, + "KL/mean": -925.3953857421875, + "KL/rejected_KL_mean": -1142.161376953125, + "KL/std": 603.607666015625, + "epoch": 0.7782672540381792, + "fcm_dpo/beta": 0.0011045248247683048, + "fcm_dpo/delta": -0.0807407796382904, + "fcm_dpo/margin": 433.53204345703125, + "fcm_dpo/q_t": 0.39779338240623474, + "grad_norm": 48.64304733276367, + "learning_rate": 7.231818622338822e-08, + "logits/chosen": -0.9852283000946045, + "logits/rejected": -0.9839862585067749, + "logps/chosen": -756.2308349609375, + "logps/ref_chosen": -47.601417541503906, + "logps/ref_rejected": -87.2845230102539, + "logps/rejected": -1229.4459228515625, + "loss": 1.1157, + "margin_dpo/margin_mean": 433.53204345703125, + "margin_dpo/margin_std": 764.8116455078125, + "step": 530 + }, + { + "KL/chosen_KL_mean": -784.3384399414062, + "KL/mean": -969.3201904296875, + "KL/rejected_KL_mean": -1154.302001953125, + "KL/std": 622.963623046875, + "epoch": 0.7797356828193832, + "fcm_dpo/beta": 0.0011020496021956205, + "fcm_dpo/delta": -0.007951600477099419, + "fcm_dpo/margin": 369.9634704589844, + "fcm_dpo/q_t": 0.4096531271934509, + "grad_norm": 47.655757904052734, + "learning_rate": 7.141774982445147e-08, + "logits/chosen": -1.1287320852279663, + "logits/rejected": -1.113837480545044, + "logps/chosen": -839.58447265625, + "logps/ref_chosen": -55.246063232421875, + "logps/ref_rejected": -70.60598754882812, + "logps/rejected": -1224.907958984375, + "loss": 1.1196, + "margin_dpo/margin_mean": 369.9634704589844, + "margin_dpo/margin_std": 602.9121704101562, + "step": 531 + }, + { + "KL/chosen_KL_mean": -803.9302368164062, + "KL/mean": -986.0310668945312, + "KL/rejected_KL_mean": -1168.1319580078125, + "KL/std": 576.912353515625, + "epoch": 0.7812041116005873, + "fcm_dpo/beta": 0.0010961306979879737, + "fcm_dpo/delta": 0.0002321675419807434, + "fcm_dpo/margin": 364.2017822265625, + "fcm_dpo/q_t": 0.4125855565071106, + "grad_norm": 69.55883026123047, + "learning_rate": 7.052201923388953e-08, + "logits/chosen": -1.0360071659088135, + "logits/rejected": -1.0059716701507568, + "logps/chosen": -874.2162475585938, + "logps/ref_chosen": -70.28601837158203, + "logps/ref_rejected": -86.5913314819336, + "logps/rejected": -1254.7232666015625, + "loss": 1.1623, + "margin_dpo/margin_mean": 364.2017517089844, + "margin_dpo/margin_std": 691.018310546875, + "step": 532 + }, + { + "KL/chosen_KL_mean": -728.8594360351562, + "KL/mean": -868.2634887695312, + "KL/rejected_KL_mean": -1007.6676025390625, + "KL/std": 510.1298522949219, + "epoch": 0.7826725403817915, + "fcm_dpo/beta": 0.0010999096557497978, + "fcm_dpo/delta": -0.006392236333340406, + "fcm_dpo/margin": 278.80816650390625, + "fcm_dpo/q_t": 0.43125709891319275, + "grad_norm": 32.62123489379883, + "learning_rate": 6.963101805503646e-08, + "logits/chosen": -1.0549545288085938, + "logits/rejected": -1.0321242809295654, + "logps/chosen": -793.7145385742188, + "logps/ref_chosen": -64.8551025390625, + "logps/ref_rejected": -76.58805847167969, + "logps/rejected": -1084.255615234375, + "loss": 1.2058, + "margin_dpo/margin_mean": 278.80816650390625, + "margin_dpo/margin_std": 602.3246459960938, + "step": 533 + }, + { + "KL/chosen_KL_mean": -744.268798828125, + "KL/mean": -940.6812744140625, + "KL/rejected_KL_mean": -1137.09375, + "KL/std": 581.1688232421875, + "epoch": 0.7841409691629956, + "fcm_dpo/beta": 0.001092701917514205, + "fcm_dpo/delta": -0.030640922486782074, + "fcm_dpo/margin": 392.8249206542969, + "fcm_dpo/q_t": 0.4050602912902832, + "grad_norm": 32.57998275756836, + "learning_rate": 6.874476976660184e-08, + "logits/chosen": -1.0226364135742188, + "logits/rejected": -1.0192043781280518, + "logps/chosen": -804.38818359375, + "logps/ref_chosen": -60.119388580322266, + "logps/ref_rejected": -78.54347229003906, + "logps/rejected": -1215.63720703125, + "loss": 1.0993, + "margin_dpo/margin_mean": 392.824951171875, + "margin_dpo/margin_std": 582.9520874023438, + "step": 534 + }, + { + "KL/chosen_KL_mean": -627.1175537109375, + "KL/mean": -852.0411987304688, + "KL/rejected_KL_mean": -1076.96484375, + "KL/std": 558.7318115234375, + "epoch": 0.7856093979441997, + "fcm_dpo/beta": 0.0010921892244368792, + "fcm_dpo/delta": -0.09375444799661636, + "fcm_dpo/margin": 449.84735107421875, + "fcm_dpo/q_t": 0.3906930983066559, + "grad_norm": 33.22504806518555, + "learning_rate": 6.786329772205246e-08, + "logits/chosen": -0.9372042417526245, + "logits/rejected": -0.939862847328186, + "logps/chosen": -681.44775390625, + "logps/ref_chosen": -54.330238342285156, + "logps/ref_rejected": -96.30763244628906, + "logps/rejected": -1173.2724609375, + "loss": 1.0486, + "margin_dpo/margin_mean": 449.84735107421875, + "margin_dpo/margin_std": 577.599609375, + "step": 535 + }, + { + "KL/chosen_KL_mean": -568.00146484375, + "KL/mean": -827.193603515625, + "KL/rejected_KL_mean": -1086.3857421875, + "KL/std": 608.2098388671875, + "epoch": 0.7870778267254038, + "fcm_dpo/beta": 0.0010737399570643902, + "fcm_dpo/delta": -0.16100189089775085, + "fcm_dpo/margin": 518.3843383789062, + "fcm_dpo/q_t": 0.3829028010368347, + "grad_norm": 56.74153137207031, + "learning_rate": 6.698662514899638e-08, + "logits/chosen": -0.9405168294906616, + "logits/rejected": -0.9727465510368347, + "logps/chosen": -615.08203125, + "logps/ref_chosen": -47.08053207397461, + "logps/ref_rejected": -89.09783935546875, + "logps/rejected": -1175.483642578125, + "loss": 1.0262, + "margin_dpo/margin_mean": 518.3843383789062, + "margin_dpo/margin_std": 723.623046875, + "step": 536 + }, + { + "KL/chosen_KL_mean": -583.8455810546875, + "KL/mean": -774.345703125, + "KL/rejected_KL_mean": -964.8458251953125, + "KL/std": 504.427490234375, + "epoch": 0.788546255506608, + "fcm_dpo/beta": 0.0010713583324104548, + "fcm_dpo/delta": -0.008746057748794556, + "fcm_dpo/margin": 381.0003662109375, + "fcm_dpo/q_t": 0.4074852764606476, + "grad_norm": 44.69304275512695, + "learning_rate": 6.611477514857114e-08, + "logits/chosen": -0.9550840854644775, + "logits/rejected": -0.9326896667480469, + "logps/chosen": -641.593017578125, + "logps/ref_chosen": -57.747467041015625, + "logps/ref_rejected": -70.43838500976562, + "logps/rejected": -1035.2841796875, + "loss": 1.1209, + "margin_dpo/margin_mean": 381.0003356933594, + "margin_dpo/margin_std": 604.8282470703125, + "step": 537 + }, + { + "KL/chosen_KL_mean": -731.8800048828125, + "KL/mean": -923.4337158203125, + "KL/rejected_KL_mean": -1114.9873046875, + "KL/std": 585.79248046875, + "epoch": 0.7900146842878121, + "fcm_dpo/beta": 0.0010648441966623068, + "fcm_dpo/delta": -0.008301455527544022, + "fcm_dpo/margin": 383.10736083984375, + "fcm_dpo/q_t": 0.4077230989933014, + "grad_norm": 41.17292785644531, + "learning_rate": 6.524777069483525e-08, + "logits/chosen": -0.9506703019142151, + "logits/rejected": -0.9313552379608154, + "logps/chosen": -798.2959594726562, + "logps/ref_chosen": -66.41594696044922, + "logps/ref_rejected": -84.22808837890625, + "logps/rejected": -1199.2154541015625, + "loss": 1.1046, + "margin_dpo/margin_mean": 383.1073913574219, + "margin_dpo/margin_std": 566.696044921875, + "step": 538 + }, + { + "KL/chosen_KL_mean": -605.9473876953125, + "KL/mean": -802.4550170898438, + "KL/rejected_KL_mean": -998.962646484375, + "KL/std": 455.50811767578125, + "epoch": 0.7914831130690162, + "fcm_dpo/beta": 0.001066508935764432, + "fcm_dpo/delta": -0.01959817484021187, + "fcm_dpo/margin": 393.01519775390625, + "fcm_dpo/q_t": 0.40525415539741516, + "grad_norm": 32.76362228393555, + "learning_rate": 6.438563463416221e-08, + "logits/chosen": -0.982710599899292, + "logits/rejected": -0.9756343364715576, + "logps/chosen": -664.4402465820312, + "logps/ref_chosen": -58.492855072021484, + "logps/ref_rejected": -91.85395050048828, + "logps/rejected": -1090.816650390625, + "loss": 1.08, + "margin_dpo/margin_mean": 393.0152282714844, + "margin_dpo/margin_std": 511.8883972167969, + "step": 539 + }, + { + "KL/chosen_KL_mean": -607.8150024414062, + "KL/mean": -850.8222045898438, + "KL/rejected_KL_mean": -1093.829345703125, + "KL/std": 560.0186767578125, + "epoch": 0.7929515418502202, + "fcm_dpo/beta": 0.0010573212057352066, + "fcm_dpo/delta": -0.11679581552743912, + "fcm_dpo/margin": 486.01434326171875, + "fcm_dpo/q_t": 0.38948220014572144, + "grad_norm": 36.21888732910156, + "learning_rate": 6.352838968463919e-08, + "logits/chosen": -0.9439930319786072, + "logits/rejected": -0.96489417552948, + "logps/chosen": -671.2975463867188, + "logps/ref_chosen": -63.482513427734375, + "logps/ref_rejected": -116.42999267578125, + "logps/rejected": -1210.25927734375, + "loss": 1.0536, + "margin_dpo/margin_mean": 486.0143127441406, + "margin_dpo/margin_std": 680.3463134765625, + "step": 540 + }, + { + "KL/chosen_KL_mean": -727.278564453125, + "KL/mean": -857.4025268554688, + "KL/rejected_KL_mean": -987.5263671875, + "KL/std": 491.0820007324219, + "epoch": 0.7944199706314243, + "fcm_dpo/beta": 0.001053128275088966, + "fcm_dpo/delta": 0.007406635209918022, + "fcm_dpo/margin": 260.24786376953125, + "fcm_dpo/q_t": 0.43850696086883545, + "grad_norm": 51.57310104370117, + "learning_rate": 6.267605843546767e-08, + "logits/chosen": -1.0330259799957275, + "logits/rejected": -1.0214934349060059, + "logps/chosen": -805.5589599609375, + "logps/ref_chosen": -78.28036499023438, + "logps/ref_rejected": -103.273681640625, + "logps/rejected": -1090.800048828125, + "loss": 1.2217, + "margin_dpo/margin_mean": 260.24786376953125, + "margin_dpo/margin_std": 586.0404052734375, + "step": 541 + }, + { + "KL/chosen_KL_mean": -627.02099609375, + "KL/mean": -874.4854125976562, + "KL/rejected_KL_mean": -1121.94970703125, + "KL/std": 565.48876953125, + "epoch": 0.7958883994126285, + "fcm_dpo/beta": 0.0010395846329629421, + "fcm_dpo/delta": -0.11870333552360535, + "fcm_dpo/margin": 494.9288024902344, + "fcm_dpo/q_t": 0.3869064450263977, + "grad_norm": 46.05047607421875, + "learning_rate": 6.182866334636888e-08, + "logits/chosen": -1.0332136154174805, + "logits/rejected": -1.0657650232315063, + "logps/chosen": -684.5059814453125, + "logps/ref_chosen": -57.48497009277344, + "logps/ref_rejected": -96.47506713867188, + "logps/rejected": -1218.4248046875, + "loss": 1.0441, + "margin_dpo/margin_mean": 494.9288024902344, + "margin_dpo/margin_std": 656.42431640625, + "step": 542 + }, + { + "KL/chosen_KL_mean": -702.0233764648438, + "KL/mean": -875.46875, + "KL/rejected_KL_mean": -1048.9141845703125, + "KL/std": 650.103271484375, + "epoch": 0.7973568281938326, + "fcm_dpo/beta": 0.0010423632338643074, + "fcm_dpo/delta": 0.03910157456994057, + "fcm_dpo/margin": 346.890869140625, + "fcm_dpo/q_t": 0.4295872449874878, + "grad_norm": 56.74467468261719, + "learning_rate": 6.098622674699147e-08, + "logits/chosen": -0.9723141193389893, + "logits/rejected": -1.0032031536102295, + "logps/chosen": -762.640869140625, + "logps/ref_chosen": -60.61750793457031, + "logps/ref_rejected": -105.59896850585938, + "logps/rejected": -1154.51318359375, + "loss": 1.199, + "margin_dpo/margin_mean": 346.890869140625, + "margin_dpo/margin_std": 787.779541015625, + "step": 543 + }, + { + "KL/chosen_KL_mean": -699.0665893554688, + "KL/mean": -904.1417236328125, + "KL/rejected_KL_mean": -1109.217041015625, + "KL/std": 525.4854125976562, + "epoch": 0.7988252569750367, + "fcm_dpo/beta": 0.0010421974584460258, + "fcm_dpo/delta": -0.028073592111468315, + "fcm_dpo/margin": 410.15032958984375, + "fcm_dpo/q_t": 0.4034753441810608, + "grad_norm": 39.14487838745117, + "learning_rate": 6.01487708363232e-08, + "logits/chosen": -0.9627532958984375, + "logits/rejected": -0.9823678135871887, + "logps/chosen": -758.7088623046875, + "logps/ref_chosen": -59.642303466796875, + "logps/ref_rejected": -100.95469665527344, + "logps/rejected": -1210.171630859375, + "loss": 1.0915, + "margin_dpo/margin_mean": 410.15032958984375, + "margin_dpo/margin_std": 594.836669921875, + "step": 544 + }, + { + "KL/chosen_KL_mean": -665.3682861328125, + "KL/mean": -904.0440063476562, + "KL/rejected_KL_mean": -1142.7197265625, + "KL/std": 538.8303833007812, + "epoch": 0.8002936857562408, + "fcm_dpo/beta": 0.0010349673684686422, + "fcm_dpo/delta": -0.0963834896683693, + "fcm_dpo/margin": 477.3514709472656, + "fcm_dpo/q_t": 0.38999858498573303, + "grad_norm": 32.36417007446289, + "learning_rate": 5.9316317682106294e-08, + "logits/chosen": -0.8844515085220337, + "logits/rejected": -0.9089665412902832, + "logps/chosen": -733.016845703125, + "logps/ref_chosen": -67.64859771728516, + "logps/ref_rejected": -95.90800476074219, + "logps/rejected": -1238.6278076171875, + "loss": 1.0415, + "margin_dpo/margin_mean": 477.3514404296875, + "margin_dpo/margin_std": 607.8944091796875, + "step": 545 + }, + { + "KL/chosen_KL_mean": -620.0447998046875, + "KL/mean": -773.5802001953125, + "KL/rejected_KL_mean": -927.1156005859375, + "KL/std": 466.5025634765625, + "epoch": 0.801762114537445, + "fcm_dpo/beta": 0.001038281712681055, + "fcm_dpo/delta": 0.0824858769774437, + "fcm_dpo/margin": 307.0708923339844, + "fcm_dpo/q_t": 0.4243466258049011, + "grad_norm": 31.921030044555664, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": -0.9386703372001648, + "logits/rejected": -0.9219088554382324, + "logps/chosen": -670.7890014648438, + "logps/ref_chosen": -50.744232177734375, + "logps/ref_rejected": -81.86622619628906, + "logps/rejected": -1008.9818725585938, + "loss": 1.1562, + "margin_dpo/margin_mean": 307.0708923339844, + "margin_dpo/margin_std": 493.91357421875, + "step": 546 + }, + { + "KL/chosen_KL_mean": -614.9042358398438, + "KL/mean": -828.0643310546875, + "KL/rejected_KL_mean": -1041.2242431640625, + "KL/std": 532.6348266601562, + "epoch": 0.8032305433186491, + "fcm_dpo/beta": 0.001038061804138124, + "fcm_dpo/delta": -0.04350613057613373, + "fcm_dpo/margin": 426.320068359375, + "fcm_dpo/q_t": 0.4003497362136841, + "grad_norm": 49.95761489868164, + "learning_rate": 5.7666507254280265e-08, + "logits/chosen": -0.9308812618255615, + "logits/rejected": -0.9381631016731262, + "logps/chosen": -688.5919189453125, + "logps/ref_chosen": -73.6877212524414, + "logps/ref_rejected": -90.76136779785156, + "logps/rejected": -1131.985595703125, + "loss": 1.0764, + "margin_dpo/margin_mean": 426.320068359375, + "margin_dpo/margin_std": 586.7791748046875, + "step": 547 + }, + { + "KL/chosen_KL_mean": -657.8882446289062, + "KL/mean": -843.9827270507812, + "KL/rejected_KL_mean": -1030.0771484375, + "KL/std": 555.6254272460938, + "epoch": 0.8046989720998532, + "fcm_dpo/beta": 0.0010367643553763628, + "fcm_dpo/delta": 0.014406869187951088, + "fcm_dpo/margin": 372.18896484375, + "fcm_dpo/q_t": 0.4164394736289978, + "grad_norm": 30.012165069580078, + "learning_rate": 5.684919345471029e-08, + "logits/chosen": -1.041344165802002, + "logits/rejected": -1.0381653308868408, + "logps/chosen": -723.1345825195312, + "logps/ref_chosen": -65.24634552001953, + "logps/ref_rejected": -94.11807250976562, + "logps/rejected": -1124.1953125, + "loss": 1.1255, + "margin_dpo/margin_mean": 372.18896484375, + "margin_dpo/margin_std": 620.2734375, + "step": 548 + }, + { + "KL/chosen_KL_mean": -680.7687377929688, + "KL/mean": -830.600341796875, + "KL/rejected_KL_mean": -980.431884765625, + "KL/std": 459.7933349609375, + "epoch": 0.8061674008810573, + "fcm_dpo/beta": 0.0010372500400990248, + "fcm_dpo/delta": -0.002494156127795577, + "fcm_dpo/margin": 299.6632080078125, + "fcm_dpo/q_t": 0.429217129945755, + "grad_norm": 53.25551223754883, + "learning_rate": 5.603696935852426e-08, + "logits/chosen": -0.9812500476837158, + "logits/rejected": -0.9636787176132202, + "logps/chosen": -729.9810791015625, + "logps/ref_chosen": -49.21235656738281, + "logps/ref_rejected": -73.91031646728516, + "logps/rejected": -1054.34228515625, + "loss": 1.1786, + "margin_dpo/margin_mean": 299.6632385253906, + "margin_dpo/margin_std": 550.6836547851562, + "step": 549 + }, + { + "KL/chosen_KL_mean": -688.97412109375, + "KL/mean": -860.2440185546875, + "KL/rejected_KL_mean": -1031.513916015625, + "KL/std": 518.7420043945312, + "epoch": 0.8076358296622613, + "fcm_dpo/beta": 0.0010400728788226843, + "fcm_dpo/delta": 0.04453900456428528, + "fcm_dpo/margin": 342.53973388671875, + "fcm_dpo/q_t": 0.41772040724754333, + "grad_norm": 38.28087615966797, + "learning_rate": 5.5229856368582376e-08, + "logits/chosen": -0.9451702833175659, + "logits/rejected": -0.9619972705841064, + "logps/chosen": -745.7811279296875, + "logps/ref_chosen": -56.80695343017578, + "logps/ref_rejected": -95.12580871582031, + "logps/rejected": -1126.6396484375, + "loss": 1.1281, + "margin_dpo/margin_mean": 342.5397644042969, + "margin_dpo/margin_std": 515.8134155273438, + "step": 550 + }, + { + "KL/chosen_KL_mean": -614.6915283203125, + "KL/mean": -889.161376953125, + "KL/rejected_KL_mean": -1163.6312255859375, + "KL/std": 555.507568359375, + "epoch": 0.8091042584434655, + "fcm_dpo/beta": 0.0010279393754899502, + "fcm_dpo/delta": -0.1691897213459015, + "fcm_dpo/margin": 548.9395751953125, + "fcm_dpo/q_t": 0.3716570734977722, + "grad_norm": 40.599369049072266, + "learning_rate": 5.4427875753062734e-08, + "logits/chosen": -0.935494065284729, + "logits/rejected": -0.987216055393219, + "logps/chosen": -673.7978515625, + "logps/ref_chosen": -59.10633087158203, + "logps/ref_rejected": -111.67280578613281, + "logps/rejected": -1275.303955078125, + "loss": 0.9718, + "margin_dpo/margin_mean": 548.939697265625, + "margin_dpo/margin_std": 550.9382934570312, + "step": 551 + }, + { + "KL/chosen_KL_mean": -578.4395751953125, + "KL/mean": -906.6759643554688, + "KL/rejected_KL_mean": -1234.912353515625, + "KL/std": 645.207763671875, + "epoch": 0.8105726872246696, + "fcm_dpo/beta": 0.000999854295514524, + "fcm_dpo/delta": -0.2669294476509094, + "fcm_dpo/margin": 656.4727783203125, + "fcm_dpo/q_t": 0.3599150478839874, + "grad_norm": 48.74257278442383, + "learning_rate": 5.363104864490034e-08, + "logits/chosen": -0.9877578020095825, + "logits/rejected": -1.0204415321350098, + "logps/chosen": -640.794189453125, + "logps/ref_chosen": -62.35459899902344, + "logps/ref_rejected": -104.56210327148438, + "logps/rejected": -1339.474365234375, + "loss": 0.9475, + "margin_dpo/margin_mean": 656.4727783203125, + "margin_dpo/margin_std": 730.9444580078125, + "step": 552 + }, + { + "KL/chosen_KL_mean": -671.1646728515625, + "KL/mean": -830.6427001953125, + "KL/rejected_KL_mean": -990.1207275390625, + "KL/std": 539.9063110351562, + "epoch": 0.8120411160058737, + "fcm_dpo/beta": 0.0010021075140684843, + "fcm_dpo/delta": 0.08171496540307999, + "fcm_dpo/margin": 318.9561462402344, + "fcm_dpo/q_t": 0.4295930862426758, + "grad_norm": 27.265270233154297, + "learning_rate": 5.2839396041230415e-08, + "logits/chosen": -0.9643408060073853, + "logits/rejected": -0.9552521109580994, + "logps/chosen": -739.4234619140625, + "logps/ref_chosen": -68.25881958007812, + "logps/ref_rejected": -98.0971450805664, + "logps/rejected": -1088.2178955078125, + "loss": 1.168, + "margin_dpo/margin_mean": 318.9561462402344, + "margin_dpo/margin_std": 580.3341064453125, + "step": 553 + }, + { + "KL/chosen_KL_mean": -698.4722900390625, + "KL/mean": -916.677978515625, + "KL/rejected_KL_mean": -1134.8837890625, + "KL/std": 573.2615966796875, + "epoch": 0.8135095447870778, + "fcm_dpo/beta": 0.0010056591127067804, + "fcm_dpo/delta": -0.04003161936998367, + "fcm_dpo/margin": 436.4113464355469, + "fcm_dpo/q_t": 0.40393197536468506, + "grad_norm": 90.09304809570312, + "learning_rate": 5.205293880283551e-08, + "logits/chosen": -0.945639967918396, + "logits/rejected": -0.9097083806991577, + "logps/chosen": -766.4199829101562, + "logps/ref_chosen": -67.94767761230469, + "logps/ref_rejected": -89.78272247314453, + "logps/rejected": -1224.66650390625, + "loss": 1.1214, + "margin_dpo/margin_mean": 436.411376953125, + "margin_dpo/margin_std": 732.8826293945312, + "step": 554 + }, + { + "KL/chosen_KL_mean": -696.947509765625, + "KL/mean": -960.1400146484375, + "KL/rejected_KL_mean": -1223.3326416015625, + "KL/std": 601.5198974609375, + "epoch": 0.8149779735682819, + "fcm_dpo/beta": 0.0009933036053553224, + "fcm_dpo/delta": -0.12606467306613922, + "fcm_dpo/margin": 526.385009765625, + "fcm_dpo/q_t": 0.3890535533428192, + "grad_norm": 34.906185150146484, + "learning_rate": 5.127169765359515e-08, + "logits/chosen": -1.0455199480056763, + "logits/rejected": -1.097933053970337, + "logps/chosen": -750.2780151367188, + "logps/ref_chosen": -53.33049011230469, + "logps/ref_rejected": -108.47937774658203, + "logps/rejected": -1331.81201171875, + "loss": 1.0568, + "margin_dpo/margin_mean": 526.385009765625, + "margin_dpo/margin_std": 771.40576171875, + "step": 555 + }, + { + "KL/chosen_KL_mean": -692.2669677734375, + "KL/mean": -844.3529052734375, + "KL/rejected_KL_mean": -996.4388427734375, + "KL/std": 467.8826599121094, + "epoch": 0.8164464023494861, + "fcm_dpo/beta": 0.000995925161987543, + "fcm_dpo/delta": 0.0985875129699707, + "fcm_dpo/margin": 304.17181396484375, + "fcm_dpo/q_t": 0.42986422777175903, + "grad_norm": 33.946044921875, + "learning_rate": 5.049569317994012e-08, + "logits/chosen": -1.021904468536377, + "logits/rejected": -1.0080738067626953, + "logps/chosen": -750.9114990234375, + "logps/ref_chosen": -58.64447021484375, + "logps/ref_rejected": -101.34040832519531, + "logps/rejected": -1097.779296875, + "loss": 1.1616, + "margin_dpo/margin_mean": 304.17181396484375, + "margin_dpo/margin_std": 485.27044677734375, + "step": 556 + }, + { + "KL/chosen_KL_mean": -720.950439453125, + "KL/mean": -956.94775390625, + "KL/rejected_KL_mean": -1192.9451904296875, + "KL/std": 639.4453735351562, + "epoch": 0.8179148311306902, + "fcm_dpo/beta": 0.000991870416328311, + "fcm_dpo/delta": -0.07001565396785736, + "fcm_dpo/margin": 471.9947204589844, + "fcm_dpo/q_t": 0.39792197942733765, + "grad_norm": 49.5215950012207, + "learning_rate": 4.9724945830310144e-08, + "logits/chosen": -1.0259175300598145, + "logits/rejected": -1.060760259628296, + "logps/chosen": -788.7911376953125, + "logps/ref_chosen": -67.84066009521484, + "logps/ref_rejected": -109.93965911865234, + "logps/rejected": -1302.884765625, + "loss": 1.0874, + "margin_dpo/margin_mean": 471.9947509765625, + "margin_dpo/margin_std": 715.3715209960938, + "step": 557 + }, + { + "KL/chosen_KL_mean": -663.1494750976562, + "KL/mean": -974.2451171875, + "KL/rejected_KL_mean": -1285.3406982421875, + "KL/std": 596.8526611328125, + "epoch": 0.8193832599118943, + "fcm_dpo/beta": 0.0009766137227416039, + "fcm_dpo/delta": -0.21403643488883972, + "fcm_dpo/margin": 622.1912841796875, + "fcm_dpo/q_t": 0.3626197576522827, + "grad_norm": 29.86164093017578, + "learning_rate": 4.8959475914614554e-08, + "logits/chosen": -1.0814244747161865, + "logits/rejected": -1.0883204936981201, + "logps/chosen": -725.5177001953125, + "logps/ref_chosen": -62.36824035644531, + "logps/ref_rejected": -102.16102600097656, + "logps/rejected": -1387.501708984375, + "loss": 0.963, + "margin_dpo/margin_mean": 622.1912841796875, + "margin_dpo/margin_std": 662.1346435546875, + "step": 558 + }, + { + "KL/chosen_KL_mean": -754.6362915039062, + "KL/mean": -1022.7030029296875, + "KL/rejected_KL_mean": -1290.7696533203125, + "KL/std": 619.6690673828125, + "epoch": 0.8208516886930984, + "fcm_dpo/beta": 0.0009626256651245058, + "fcm_dpo/delta": -0.11912199854850769, + "fcm_dpo/margin": 536.13330078125, + "fcm_dpo/q_t": 0.3861456513404846, + "grad_norm": 31.161109924316406, + "learning_rate": 4.8199303603697614e-08, + "logits/chosen": -1.1892856359481812, + "logits/rejected": -1.1905286312103271, + "logps/chosen": -815.388671875, + "logps/ref_chosen": -60.752323150634766, + "logps/ref_rejected": -93.44229125976562, + "logps/rejected": -1384.2119140625, + "loss": 1.0321, + "margin_dpo/margin_mean": 536.13330078125, + "margin_dpo/margin_std": 686.558349609375, + "step": 559 + }, + { + "KL/chosen_KL_mean": -694.3114013671875, + "KL/mean": -896.381591796875, + "KL/rejected_KL_mean": -1098.4517822265625, + "KL/std": 594.5814208984375, + "epoch": 0.8223201174743024, + "fcm_dpo/beta": 0.0009576280135661364, + "fcm_dpo/delta": 0.012987758964300156, + "fcm_dpo/margin": 404.14044189453125, + "fcm_dpo/q_t": 0.4146135449409485, + "grad_norm": 38.00310134887695, + "learning_rate": 4.7444448928806615e-08, + "logits/chosen": -0.9253185987472534, + "logits/rejected": -0.9051507711410522, + "logps/chosen": -752.4151611328125, + "logps/ref_chosen": -58.10382080078125, + "logps/ref_rejected": -79.99122619628906, + "logps/rejected": -1178.4429931640625, + "loss": 1.1291, + "margin_dpo/margin_mean": 404.14044189453125, + "margin_dpo/margin_std": 662.3407592773438, + "step": 560 + }, + { + "KL/chosen_KL_mean": -819.3272705078125, + "KL/mean": -979.8409423828125, + "KL/rejected_KL_mean": -1140.3546142578125, + "KL/std": 548.7021484375, + "epoch": 0.8237885462555066, + "fcm_dpo/beta": 0.0009685677941888571, + "fcm_dpo/delta": 0.09021740406751633, + "fcm_dpo/margin": 321.0272521972656, + "fcm_dpo/q_t": 0.4277527630329132, + "grad_norm": 40.23988723754883, + "learning_rate": 4.669493178106432e-08, + "logits/chosen": -1.1046230792999268, + "logits/rejected": -1.1246148347854614, + "logps/chosen": -870.2401733398438, + "logps/ref_chosen": -50.912879943847656, + "logps/ref_rejected": -99.06856536865234, + "logps/rejected": -1239.423095703125, + "loss": 1.2007, + "margin_dpo/margin_mean": 321.02728271484375, + "margin_dpo/margin_std": 687.259521484375, + "step": 561 + }, + { + "KL/chosen_KL_mean": -745.23193359375, + "KL/mean": -967.99609375, + "KL/rejected_KL_mean": -1190.76025390625, + "KL/std": 601.22607421875, + "epoch": 0.8252569750367107, + "fcm_dpo/beta": 0.0009640766074880958, + "fcm_dpo/delta": -0.030630581080913544, + "fcm_dpo/margin": 445.52838134765625, + "fcm_dpo/q_t": 0.404424250125885, + "grad_norm": 46.153045654296875, + "learning_rate": 4.5950771910944596e-08, + "logits/chosen": -1.0498061180114746, + "logits/rejected": -1.0524308681488037, + "logps/chosen": -804.6962890625, + "logps/ref_chosen": -59.46440124511719, + "logps/ref_rejected": -96.54266357421875, + "logps/rejected": -1287.302978515625, + "loss": 1.0927, + "margin_dpo/margin_mean": 445.5283508300781, + "margin_dpo/margin_std": 656.296630859375, + "step": 562 + }, + { + "KL/chosen_KL_mean": -846.0560913085938, + "KL/mean": -1012.888916015625, + "KL/rejected_KL_mean": -1179.7216796875, + "KL/std": 638.2447509765625, + "epoch": 0.8267254038179148, + "fcm_dpo/beta": 0.0009634558809921145, + "fcm_dpo/delta": -0.05767164006829262, + "fcm_dpo/margin": 333.6656494140625, + "fcm_dpo/q_t": 0.42173728346824646, + "grad_norm": 55.212547302246094, + "learning_rate": 4.521198892775202e-08, + "logits/chosen": -1.0601184368133545, + "logits/rejected": -1.059585690498352, + "logps/chosen": -906.664306640625, + "logps/ref_chosen": -60.60819625854492, + "logps/ref_rejected": -94.56770324707031, + "logps/rejected": -1274.2894287109375, + "loss": 1.2261, + "margin_dpo/margin_mean": 333.6656494140625, + "margin_dpo/margin_std": 764.817138671875, + "step": 563 + }, + { + "KL/chosen_KL_mean": -765.0787353515625, + "KL/mean": -982.6279296875, + "KL/rejected_KL_mean": -1200.17724609375, + "KL/std": 581.216064453125, + "epoch": 0.8281938325991189, + "fcm_dpo/beta": 0.0009602903737686574, + "fcm_dpo/delta": -0.0182206854224205, + "fcm_dpo/margin": 435.0984191894531, + "fcm_dpo/q_t": 0.4047633409500122, + "grad_norm": 36.67222595214844, + "learning_rate": 4.447860229910544e-08, + "logits/chosen": -1.1164246797561646, + "logits/rejected": -1.107216477394104, + "logps/chosen": -839.3470458984375, + "logps/ref_chosen": -74.26837921142578, + "logps/ref_rejected": -93.23818969726562, + "logps/rejected": -1293.415283203125, + "loss": 1.0771, + "margin_dpo/margin_mean": 435.0984191894531, + "margin_dpo/margin_std": 550.7127685546875, + "step": 564 + }, + { + "KL/chosen_KL_mean": -791.003173828125, + "KL/mean": -1022.08740234375, + "KL/rejected_KL_mean": -1253.171630859375, + "KL/std": 678.6461181640625, + "epoch": 0.8296622613803231, + "fcm_dpo/beta": 0.000955872586928308, + "fcm_dpo/delta": -0.04271770268678665, + "fcm_dpo/margin": 462.16839599609375, + "fcm_dpo/q_t": 0.4068571925163269, + "grad_norm": 51.650264739990234, + "learning_rate": 4.375063135042445e-08, + "logits/chosen": -1.0443668365478516, + "logits/rejected": -1.038867712020874, + "logps/chosen": -860.0230712890625, + "logps/ref_chosen": -69.0199203491211, + "logps/ref_rejected": -85.7789306640625, + "logps/rejected": -1338.9505615234375, + "loss": 1.1256, + "margin_dpo/margin_mean": 462.16839599609375, + "margin_dpo/margin_std": 813.2290649414062, + "step": 565 + }, + { + "KL/chosen_KL_mean": -779.5007934570312, + "KL/mean": -1033.7266845703125, + "KL/rejected_KL_mean": -1287.95263671875, + "KL/std": 692.0765380859375, + "epoch": 0.8311306901615272, + "fcm_dpo/beta": 0.0009514681296423078, + "fcm_dpo/delta": -0.08593890815973282, + "fcm_dpo/margin": 508.45184326171875, + "fcm_dpo/q_t": 0.39394253492355347, + "grad_norm": 33.00722885131836, + "learning_rate": 4.3028095264420525e-08, + "logits/chosen": -1.0474693775177002, + "logits/rejected": -1.0704981088638306, + "logps/chosen": -846.046142578125, + "logps/ref_chosen": -66.5453109741211, + "logps/ref_rejected": -103.86932373046875, + "logps/rejected": -1391.822021484375, + "loss": 1.0916, + "margin_dpo/margin_mean": 508.4518127441406, + "margin_dpo/margin_std": 790.054931640625, + "step": 566 + }, + { + "KL/chosen_KL_mean": -752.8310546875, + "KL/mean": -941.5557861328125, + "KL/rejected_KL_mean": -1130.2806396484375, + "KL/std": 474.6645202636719, + "epoch": 0.8325991189427313, + "fcm_dpo/beta": 0.0009493097895756364, + "fcm_dpo/delta": 0.042454224079847336, + "fcm_dpo/margin": 377.4496154785156, + "fcm_dpo/q_t": 0.416124165058136, + "grad_norm": 37.01401138305664, + "learning_rate": 4.231101308059165e-08, + "logits/chosen": -1.1754206418991089, + "logits/rejected": -1.1703362464904785, + "logps/chosen": -805.6893310546875, + "logps/ref_chosen": -52.85829544067383, + "logps/ref_rejected": -85.37095642089844, + "logps/rejected": -1215.651611328125, + "loss": 1.1146, + "margin_dpo/margin_mean": 377.4496154785156, + "margin_dpo/margin_std": 508.71038818359375, + "step": 567 + }, + { + "KL/chosen_KL_mean": -732.2120361328125, + "KL/mean": -992.5909423828125, + "KL/rejected_KL_mean": -1252.9698486328125, + "KL/std": 562.322265625, + "epoch": 0.8340675477239354, + "fcm_dpo/beta": 0.0009439606801606715, + "fcm_dpo/delta": -0.09389245510101318, + "fcm_dpo/margin": 520.7578735351562, + "fcm_dpo/q_t": 0.38713839650154114, + "grad_norm": 28.33488655090332, + "learning_rate": 4.1599403694720145e-08, + "logits/chosen": -1.0602999925613403, + "logits/rejected": -1.0991747379302979, + "logps/chosen": -777.4044189453125, + "logps/ref_chosen": -45.1923828125, + "logps/ref_rejected": -89.09236907958984, + "logps/rejected": -1342.062255859375, + "loss": 1.0249, + "margin_dpo/margin_mean": 520.7578125, + "margin_dpo/margin_std": 590.0891723632812, + "step": 568 + }, + { + "KL/chosen_KL_mean": -826.0765380859375, + "KL/mean": -1036.68115234375, + "KL/rejected_KL_mean": -1247.285888671875, + "KL/std": 716.984130859375, + "epoch": 0.8355359765051396, + "fcm_dpo/beta": 0.000945072213653475, + "fcm_dpo/delta": 0.0015114173293113708, + "fcm_dpo/margin": 421.209228515625, + "fcm_dpo/q_t": 0.4105045795440674, + "grad_norm": 47.668739318847656, + "learning_rate": 4.089328585837512e-08, + "logits/chosen": -1.1122533082962036, + "logits/rejected": -1.1140843629837036, + "logps/chosen": -889.797119140625, + "logps/ref_chosen": -63.72056198120117, + "logps/ref_rejected": -79.10325622558594, + "logps/rejected": -1326.3890380859375, + "loss": 1.1478, + "margin_dpo/margin_mean": 421.2092590332031, + "margin_dpo/margin_std": 756.34326171875, + "step": 569 + }, + { + "KL/chosen_KL_mean": -775.0736083984375, + "KL/mean": -970.983154296875, + "KL/rejected_KL_mean": -1166.8927001953125, + "KL/std": 585.8502197265625, + "epoch": 0.8370044052863436, + "fcm_dpo/beta": 0.0009430573554709554, + "fcm_dpo/delta": 0.031048066914081573, + "fcm_dpo/margin": 391.8191223144531, + "fcm_dpo/q_t": 0.4179254174232483, + "grad_norm": 31.495166778564453, + "learning_rate": 4.019267817841834e-08, + "logits/chosen": -1.1752269268035889, + "logits/rejected": -1.1550638675689697, + "logps/chosen": -836.6881103515625, + "logps/ref_chosen": -61.61454391479492, + "logps/ref_rejected": -82.14186096191406, + "logps/rejected": -1249.03466796875, + "loss": 1.1347, + "margin_dpo/margin_mean": 391.819091796875, + "margin_dpo/margin_std": 633.9605102539062, + "step": 570 + }, + { + "KL/chosen_KL_mean": -785.480224609375, + "KL/mean": -1001.910888671875, + "KL/rejected_KL_mean": -1218.341552734375, + "KL/std": 558.670166015625, + "epoch": 0.8384728340675477, + "fcm_dpo/beta": 0.0009431964717805386, + "fcm_dpo/delta": -0.008509235456585884, + "fcm_dpo/margin": 432.861328125, + "fcm_dpo/q_t": 0.40848302841186523, + "grad_norm": 44.37576675415039, + "learning_rate": 3.9497599116513705e-08, + "logits/chosen": -1.1002910137176514, + "logits/rejected": -1.1096596717834473, + "logps/chosen": -838.5343017578125, + "logps/ref_chosen": -53.05406188964844, + "logps/ref_rejected": -91.33682250976562, + "logps/rejected": -1309.678466796875, + "loss": 1.1233, + "margin_dpo/margin_mean": 432.861328125, + "margin_dpo/margin_std": 713.20458984375, + "step": 571 + }, + { + "KL/chosen_KL_mean": -822.61279296875, + "KL/mean": -1061.8223876953125, + "KL/rejected_KL_mean": -1301.031982421875, + "KL/std": 679.26806640625, + "epoch": 0.8399412628487518, + "fcm_dpo/beta": 0.0009392797946929932, + "fcm_dpo/delta": -0.05064094066619873, + "fcm_dpo/margin": 478.4192810058594, + "fcm_dpo/q_t": 0.40520572662353516, + "grad_norm": 29.390687942504883, + "learning_rate": 3.880806698864086e-08, + "logits/chosen": -1.128666877746582, + "logits/rejected": -1.1588270664215088, + "logps/chosen": -871.0720825195312, + "logps/ref_chosen": -48.45928955078125, + "logps/ref_rejected": -83.55703735351562, + "logps/rejected": -1384.589111328125, + "loss": 1.1148, + "margin_dpo/margin_mean": 478.41925048828125, + "margin_dpo/margin_std": 823.0018310546875, + "step": 572 + }, + { + "KL/chosen_KL_mean": -781.2991943359375, + "KL/mean": -1001.1578979492188, + "KL/rejected_KL_mean": -1221.0164794921875, + "KL/std": 596.3881225585938, + "epoch": 0.8414096916299559, + "fcm_dpo/beta": 0.0009392928332090378, + "fcm_dpo/delta": -0.013323244638741016, + "fcm_dpo/margin": 439.71728515625, + "fcm_dpo/q_t": 0.40707600116729736, + "grad_norm": 29.680734634399414, + "learning_rate": 3.812409996461275e-08, + "logits/chosen": -1.162482738494873, + "logits/rejected": -1.1760870218276978, + "logps/chosen": -832.921875, + "logps/ref_chosen": -51.62262725830078, + "logps/ref_rejected": -85.32499694824219, + "logps/rejected": -1306.3414306640625, + "loss": 1.0857, + "margin_dpo/margin_mean": 439.71728515625, + "margin_dpo/margin_std": 595.1873779296875, + "step": 573 + }, + { + "KL/chosen_KL_mean": -724.2657470703125, + "KL/mean": -953.5592041015625, + "KL/rejected_KL_mean": -1182.8525390625, + "KL/std": 532.4700927734375, + "epoch": 0.8428781204111601, + "fcm_dpo/beta": 0.0009368173778057098, + "fcm_dpo/delta": -0.030269186943769455, + "fcm_dpo/margin": 458.5869140625, + "fcm_dpo/q_t": 0.40228691697120667, + "grad_norm": 29.51076889038086, + "learning_rate": 3.74457160675965e-08, + "logits/chosen": -1.1240806579589844, + "logits/rejected": -1.145003318786621, + "logps/chosen": -775.3101806640625, + "logps/ref_chosen": -51.04446029663086, + "logps/ref_rejected": -92.80640411376953, + "logps/rejected": -1275.658935546875, + "loss": 1.0787, + "margin_dpo/margin_mean": 458.5869140625, + "margin_dpo/margin_std": 613.4527587890625, + "step": 574 + }, + { + "KL/chosen_KL_mean": -811.00634765625, + "KL/mean": -1024.9534912109375, + "KL/rejected_KL_mean": -1238.9005126953125, + "KL/std": 559.50830078125, + "epoch": 0.8443465491923642, + "fcm_dpo/beta": 0.0009320209501311183, + "fcm_dpo/delta": 0.000727042555809021, + "fcm_dpo/margin": 427.89422607421875, + "fcm_dpo/q_t": 0.4095437228679657, + "grad_norm": 42.62068557739258, + "learning_rate": 3.677293317363864e-08, + "logits/chosen": -1.046234130859375, + "logits/rejected": -1.0514554977416992, + "logps/chosen": -882.7965087890625, + "logps/ref_chosen": -71.7901382446289, + "logps/ref_rejected": -95.38619995117188, + "logps/rejected": -1334.2867431640625, + "loss": 1.1328, + "margin_dpo/margin_mean": 427.89422607421875, + "margin_dpo/margin_std": 713.029541015625, + "step": 575 + }, + { + "KL/chosen_KL_mean": -782.13525390625, + "KL/mean": -943.5439453125, + "KL/rejected_KL_mean": -1104.95263671875, + "KL/std": 511.6619567871094, + "epoch": 0.8458149779735683, + "fcm_dpo/beta": 0.0009402718860656023, + "fcm_dpo/delta": 0.09798312187194824, + "fcm_dpo/margin": 322.81732177734375, + "fcm_dpo/q_t": 0.429963618516922, + "grad_norm": 39.150245666503906, + "learning_rate": 3.6105769011194224e-08, + "logits/chosen": -1.1449861526489258, + "logits/rejected": -1.1737542152404785, + "logps/chosen": -836.3982543945312, + "logps/ref_chosen": -54.262962341308594, + "logps/ref_rejected": -100.75428009033203, + "logps/rejected": -1205.7069091796875, + "loss": 1.1796, + "margin_dpo/margin_mean": 322.8173828125, + "margin_dpo/margin_std": 582.9664306640625, + "step": 576 + }, + { + "KL/chosen_KL_mean": -690.1397094726562, + "KL/mean": -899.7841186523438, + "KL/rejected_KL_mean": -1109.428466796875, + "KL/std": 570.5064086914062, + "epoch": 0.8472834067547724, + "fcm_dpo/beta": 0.0009443428134545684, + "fcm_dpo/delta": 0.004122039768844843, + "fcm_dpo/margin": 419.2888488769531, + "fcm_dpo/q_t": 0.4105992913246155, + "grad_norm": 30.886186599731445, + "learning_rate": 3.5444241160659304e-08, + "logits/chosen": -1.102234125137329, + "logits/rejected": -1.082305669784546, + "logps/chosen": -752.0494384765625, + "logps/ref_chosen": -61.909706115722656, + "logps/ref_rejected": -84.07069396972656, + "logps/rejected": -1193.499267578125, + "loss": 1.1191, + "margin_dpo/margin_mean": 419.28887939453125, + "margin_dpo/margin_std": 626.2908325195312, + "step": 577 + }, + { + "KL/chosen_KL_mean": -685.677978515625, + "KL/mean": -919.28759765625, + "KL/rejected_KL_mean": -1152.897216796875, + "KL/std": 582.769287109375, + "epoch": 0.8487518355359766, + "fcm_dpo/beta": 0.0009395558736287057, + "fcm_dpo/delta": -0.04024638235569, + "fcm_dpo/margin": 467.21929931640625, + "fcm_dpo/q_t": 0.39982476830482483, + "grad_norm": 37.12548065185547, + "learning_rate": 3.478836705390808e-08, + "logits/chosen": -0.9650574922561646, + "logits/rejected": -0.9971319437026978, + "logps/chosen": -734.941650390625, + "logps/ref_chosen": -49.26368713378906, + "logps/ref_rejected": -83.4362564086914, + "logps/rejected": -1236.33349609375, + "loss": 1.0655, + "margin_dpo/margin_mean": 467.21929931640625, + "margin_dpo/margin_std": 579.5484619140625, + "step": 578 + }, + { + "KL/chosen_KL_mean": -784.9281005859375, + "KL/mean": -929.6224365234375, + "KL/rejected_KL_mean": -1074.316650390625, + "KL/std": 574.9615478515625, + "epoch": 0.8502202643171806, + "fcm_dpo/beta": 0.0009488767245784402, + "fcm_dpo/delta": 0.1272491067647934, + "fcm_dpo/margin": 289.38848876953125, + "fcm_dpo/q_t": 0.4369036555290222, + "grad_norm": 43.24967956542969, + "learning_rate": 3.41381639738331e-08, + "logits/chosen": -1.0656390190124512, + "logits/rejected": -1.0647817850112915, + "logps/chosen": -843.81396484375, + "logps/ref_chosen": -58.88581848144531, + "logps/ref_rejected": -94.78762817382812, + "logps/rejected": -1169.104248046875, + "loss": 1.2174, + "margin_dpo/margin_mean": 289.3885192871094, + "margin_dpo/margin_std": 634.266845703125, + "step": 579 + }, + { + "KL/chosen_KL_mean": -582.00390625, + "KL/mean": -842.2623901367188, + "KL/rejected_KL_mean": -1102.5208740234375, + "KL/std": 623.9799194335938, + "epoch": 0.8516886930983847, + "fcm_dpo/beta": 0.0009446794865652919, + "fcm_dpo/delta": -0.09435415267944336, + "fcm_dpo/margin": 520.5169677734375, + "fcm_dpo/q_t": 0.39273136854171753, + "grad_norm": 27.779617309570312, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": -0.8903079032897949, + "logits/rejected": -0.9164822101593018, + "logps/chosen": -630.7107543945312, + "logps/ref_chosen": -48.70683670043945, + "logps/ref_rejected": -81.7583999633789, + "logps/rejected": -1184.279296875, + "loss": 1.049, + "margin_dpo/margin_mean": 520.5169677734375, + "margin_dpo/margin_std": 708.4932861328125, + "step": 580 + }, + { + "KL/chosen_KL_mean": -791.675048828125, + "KL/mean": -977.7584228515625, + "KL/rejected_KL_mean": -1163.841796875, + "KL/std": 612.6318359375, + "epoch": 0.8531571218795888, + "fcm_dpo/beta": 0.0009482804453000426, + "fcm_dpo/delta": 0.04787049442529678, + "fcm_dpo/margin": 372.1667175292969, + "fcm_dpo/q_t": 0.4224420487880707, + "grad_norm": 41.1692008972168, + "learning_rate": 3.285483927764726e-08, + "logits/chosen": -1.1497280597686768, + "logits/rejected": -1.1572540998458862, + "logps/chosen": -853.8973999023438, + "logps/ref_chosen": -62.22235107421875, + "logps/ref_rejected": -91.73568725585938, + "logps/rejected": -1255.577392578125, + "loss": 1.154, + "margin_dpo/margin_mean": 372.166748046875, + "margin_dpo/margin_std": 670.7994995117188, + "step": 581 + }, + { + "KL/chosen_KL_mean": -701.05224609375, + "KL/mean": -898.874755859375, + "KL/rejected_KL_mean": -1096.6973876953125, + "KL/std": 523.3825073242188, + "epoch": 0.8546255506607929, + "fcm_dpo/beta": 0.0009447854827158153, + "fcm_dpo/delta": -0.08350235223770142, + "fcm_dpo/margin": 395.6450500488281, + "fcm_dpo/q_t": 0.4139753580093384, + "grad_norm": 35.411434173583984, + "learning_rate": 3.222175147833556e-08, + "logits/chosen": -1.082298994064331, + "logits/rejected": -1.106623649597168, + "logps/chosen": -759.2808837890625, + "logps/ref_chosen": -58.228660583496094, + "logps/ref_rejected": -110.06959533691406, + "logps/rejected": -1206.7669677734375, + "loss": 1.1245, + "margin_dpo/margin_mean": 395.6450500488281, + "margin_dpo/margin_std": 574.4161376953125, + "step": 582 + }, + { + "KL/chosen_KL_mean": -788.7952880859375, + "KL/mean": -915.5237426757812, + "KL/rejected_KL_mean": -1042.252197265625, + "KL/std": 566.1787109375, + "epoch": 0.856093979441997, + "fcm_dpo/beta": 0.0009400502895005047, + "fcm_dpo/delta": -0.008407074958086014, + "fcm_dpo/margin": 253.45693969726562, + "fcm_dpo/q_t": 0.44672808051109314, + "grad_norm": 57.46125411987305, + "learning_rate": 3.159440233840763e-08, + "logits/chosen": -0.9853731393814087, + "logits/rejected": -0.978585422039032, + "logps/chosen": -845.6581420898438, + "logps/ref_chosen": -56.86286163330078, + "logps/ref_rejected": -88.4039306640625, + "logps/rejected": -1130.6561279296875, + "loss": 1.2656, + "margin_dpo/margin_mean": 253.45692443847656, + "margin_dpo/margin_std": 682.3837890625, + "step": 583 + }, + { + "KL/chosen_KL_mean": -670.3692626953125, + "KL/mean": -927.6574096679688, + "KL/rejected_KL_mean": -1184.945556640625, + "KL/std": 594.9502563476562, + "epoch": 0.8575624082232012, + "fcm_dpo/beta": 0.000934588722884655, + "fcm_dpo/delta": -0.0828862339258194, + "fcm_dpo/margin": 514.5762939453125, + "fcm_dpo/q_t": 0.39284205436706543, + "grad_norm": 29.62038230895996, + "learning_rate": 3.0972808389096635e-08, + "logits/chosen": -1.0637917518615723, + "logits/rejected": -1.0736515522003174, + "logps/chosen": -727.2699584960938, + "logps/ref_chosen": -56.90068054199219, + "logps/ref_rejected": -97.63606262207031, + "logps/rejected": -1282.5816650390625, + "loss": 1.0451, + "margin_dpo/margin_mean": 514.5762939453125, + "margin_dpo/margin_std": 644.3516845703125, + "step": 584 + }, + { + "KL/chosen_KL_mean": -772.5379638671875, + "KL/mean": -1022.5894775390625, + "KL/rejected_KL_mean": -1272.6409912109375, + "KL/std": 647.20556640625, + "epoch": 0.8590308370044053, + "fcm_dpo/beta": 0.0009266703855246305, + "fcm_dpo/delta": -0.06506932526826859, + "fcm_dpo/margin": 500.10296630859375, + "fcm_dpo/q_t": 0.3975642919540405, + "grad_norm": 34.582088470458984, + "learning_rate": 3.035698600998121e-08, + "logits/chosen": -1.089249849319458, + "logits/rejected": -1.112525224685669, + "logps/chosen": -833.511962890625, + "logps/ref_chosen": -60.973968505859375, + "logps/ref_rejected": -84.16952514648438, + "logps/rejected": -1356.810546875, + "loss": 1.0897, + "margin_dpo/margin_mean": 500.10296630859375, + "margin_dpo/margin_std": 768.308349609375, + "step": 585 + }, + { + "KL/chosen_KL_mean": -807.0784912109375, + "KL/mean": -980.6554565429688, + "KL/rejected_KL_mean": -1154.232421875, + "KL/std": 573.5162963867188, + "epoch": 0.8604992657856094, + "fcm_dpo/beta": 0.0009305774001404643, + "fcm_dpo/delta": 0.07824563980102539, + "fcm_dpo/margin": 347.15399169921875, + "fcm_dpo/q_t": 0.42543265223503113, + "grad_norm": 33.32202911376953, + "learning_rate": 2.974695142855388e-08, + "logits/chosen": -1.1124446392059326, + "logits/rejected": -1.137439250946045, + "logps/chosen": -863.93408203125, + "logps/ref_chosen": -56.85559844970703, + "logps/ref_rejected": -91.80261993408203, + "logps/rejected": -1246.03515625, + "loss": 1.1743, + "margin_dpo/margin_mean": 347.15399169921875, + "margin_dpo/margin_std": 641.89892578125, + "step": 586 + }, + { + "KL/chosen_KL_mean": -590.2689208984375, + "KL/mean": -805.8556518554688, + "KL/rejected_KL_mean": -1021.4425048828125, + "KL/std": 601.960205078125, + "epoch": 0.8619676945668135, + "fcm_dpo/beta": 0.0009339781245216727, + "fcm_dpo/delta": -0.002780054695904255, + "fcm_dpo/margin": 431.173583984375, + "fcm_dpo/q_t": 0.4075835347175598, + "grad_norm": 36.960052490234375, + "learning_rate": 2.9142720719793122e-08, + "logits/chosen": -1.1249654293060303, + "logits/rejected": -1.1489168405532837, + "logps/chosen": -634.96044921875, + "logps/ref_chosen": -44.69159698486328, + "logps/ref_rejected": -82.62385559082031, + "logps/rejected": -1104.06640625, + "loss": 1.0941, + "margin_dpo/margin_mean": 431.1735534667969, + "margin_dpo/margin_std": 591.1593017578125, + "step": 587 + }, + { + "KL/chosen_KL_mean": -746.4495849609375, + "KL/mean": -930.6257934570312, + "KL/rejected_KL_mean": -1114.802001953125, + "KL/std": 514.0170288085938, + "epoch": 0.8634361233480177, + "fcm_dpo/beta": 0.0009352130000479519, + "fcm_dpo/delta": 0.05642148479819298, + "fcm_dpo/margin": 368.352294921875, + "fcm_dpo/q_t": 0.4200473427772522, + "grad_norm": 30.264469146728516, + "learning_rate": 2.8544309805740018e-08, + "logits/chosen": -1.071812391281128, + "logits/rejected": -1.0980116128921509, + "logps/chosen": -796.7445678710938, + "logps/ref_chosen": -50.29494857788086, + "logps/ref_rejected": -107.36988067626953, + "logps/rejected": -1222.171875, + "loss": 1.1337, + "margin_dpo/margin_mean": 368.3523254394531, + "margin_dpo/margin_std": 540.8074340820312, + "step": 588 + }, + { + "KL/chosen_KL_mean": -765.505615234375, + "KL/mean": -1002.3455810546875, + "KL/rejected_KL_mean": -1239.1854248046875, + "KL/std": 586.9735717773438, + "epoch": 0.8649045521292217, + "fcm_dpo/beta": 0.0009358683601021767, + "fcm_dpo/delta": -0.04427178204059601, + "fcm_dpo/margin": 473.67987060546875, + "fcm_dpo/q_t": 0.3995531499385834, + "grad_norm": 30.180952072143555, + "learning_rate": 2.7951734455078786e-08, + "logits/chosen": -1.046541690826416, + "logits/rejected": -1.0619137287139893, + "logps/chosen": -825.435546875, + "logps/ref_chosen": -59.929908752441406, + "logps/ref_rejected": -111.65534973144531, + "logps/rejected": -1350.8408203125, + "loss": 1.071, + "margin_dpo/margin_mean": 473.67987060546875, + "margin_dpo/margin_std": 629.702392578125, + "step": 589 + }, + { + "KL/chosen_KL_mean": -648.0178833007812, + "KL/mean": -879.0958862304688, + "KL/rejected_KL_mean": -1110.173828125, + "KL/std": 540.598388671875, + "epoch": 0.8663729809104258, + "fcm_dpo/beta": 0.0009314791532233357, + "fcm_dpo/delta": -0.031209638342261314, + "fcm_dpo/margin": 462.15606689453125, + "fcm_dpo/q_t": 0.40161752700805664, + "grad_norm": 23.975587844848633, + "learning_rate": 2.736501028272095e-08, + "logits/chosen": -1.0303199291229248, + "logits/rejected": -1.0523946285247803, + "logps/chosen": -703.82763671875, + "logps/ref_chosen": -55.80979537963867, + "logps/ref_rejected": -106.06282043457031, + "logps/rejected": -1216.23681640625, + "loss": 1.0681, + "margin_dpo/margin_mean": 462.1560363769531, + "margin_dpo/margin_std": 576.578125, + "step": 590 + }, + { + "KL/chosen_KL_mean": -737.061767578125, + "KL/mean": -956.1275024414062, + "KL/rejected_KL_mean": -1175.1932373046875, + "KL/std": 550.1436157226562, + "epoch": 0.8678414096916299, + "fcm_dpo/beta": 0.0009308913140557706, + "fcm_dpo/delta": -0.008016519248485565, + "fcm_dpo/margin": 438.1313781738281, + "fcm_dpo/q_t": 0.4057263135910034, + "grad_norm": 30.565818786621094, + "learning_rate": 2.678415274939408e-08, + "logits/chosen": -1.1124975681304932, + "logits/rejected": -1.0994548797607422, + "logps/chosen": -793.3024291992188, + "logps/ref_chosen": -56.24061965942383, + "logps/ref_rejected": -83.78629302978516, + "logps/rejected": -1258.9794921875, + "loss": 1.0962, + "margin_dpo/margin_mean": 438.13140869140625, + "margin_dpo/margin_std": 615.4335327148438, + "step": 591 + }, + { + "KL/chosen_KL_mean": -743.1383666992188, + "KL/mean": -941.9227905273438, + "KL/rejected_KL_mean": -1140.707275390625, + "KL/std": 541.6781616210938, + "epoch": 0.869309838472834, + "fcm_dpo/beta": 0.0009322206024080515, + "fcm_dpo/delta": 0.029935326427221298, + "fcm_dpo/margin": 397.5688171386719, + "fcm_dpo/q_t": 0.416223406791687, + "grad_norm": 30.50063705444336, + "learning_rate": 2.6209177161234442e-08, + "logits/chosen": -1.1092453002929688, + "logits/rejected": -1.1124801635742188, + "logps/chosen": -791.07861328125, + "logps/ref_chosen": -47.94025421142578, + "logps/ref_rejected": -75.73287963867188, + "logps/rejected": -1216.440185546875, + "loss": 1.1635, + "margin_dpo/margin_mean": 397.56884765625, + "margin_dpo/margin_std": 738.2605590820312, + "step": 592 + }, + { + "KL/chosen_KL_mean": -737.1370849609375, + "KL/mean": -894.179931640625, + "KL/rejected_KL_mean": -1051.2227783203125, + "KL/std": 630.5286865234375, + "epoch": 0.8707782672540382, + "fcm_dpo/beta": 0.0009341684053651989, + "fcm_dpo/delta": 0.006865399423986673, + "fcm_dpo/margin": 314.085693359375, + "fcm_dpo/q_t": 0.4349074959754944, + "grad_norm": 45.077449798583984, + "learning_rate": 2.564009866938349e-08, + "logits/chosen": -0.925495982170105, + "logits/rejected": -0.9140543937683105, + "logps/chosen": -785.827880859375, + "logps/ref_chosen": -48.690757751464844, + "logps/ref_rejected": -60.90800094604492, + "logps/rejected": -1112.130859375, + "loss": 1.2123, + "margin_dpo/margin_mean": 314.085693359375, + "margin_dpo/margin_std": 687.1311645507812, + "step": 593 + }, + { + "KL/chosen_KL_mean": -696.455810546875, + "KL/mean": -892.4623413085938, + "KL/rejected_KL_mean": -1088.468994140625, + "KL/std": 599.05810546875, + "epoch": 0.8722466960352423, + "fcm_dpo/beta": 0.0009395517408847809, + "fcm_dpo/delta": 0.03171183168888092, + "fcm_dpo/margin": 392.01312255859375, + "fcm_dpo/q_t": 0.4168938100337982, + "grad_norm": 32.01091003417969, + "learning_rate": 2.5076932269588708e-08, + "logits/chosen": -1.0287201404571533, + "logits/rejected": -1.0127062797546387, + "logps/chosen": -751.3906860351562, + "logps/ref_chosen": -54.93488693237305, + "logps/ref_rejected": -86.09967803955078, + "logps/rejected": -1174.568603515625, + "loss": 1.1387, + "margin_dpo/margin_mean": 392.01312255859375, + "margin_dpo/margin_std": 633.80126953125, + "step": 594 + }, + { + "KL/chosen_KL_mean": -645.5115356445312, + "KL/mean": -869.5858154296875, + "KL/rejected_KL_mean": -1093.66015625, + "KL/std": 564.3441772460938, + "epoch": 0.8737151248164464, + "fcm_dpo/beta": 0.0009344806894659996, + "fcm_dpo/delta": -0.01929367519915104, + "fcm_dpo/margin": 448.14862060546875, + "fcm_dpo/q_t": 0.40738070011138916, + "grad_norm": 33.268638610839844, + "learning_rate": 2.451969280180849e-08, + "logits/chosen": -1.0355620384216309, + "logits/rejected": -1.0576274394989014, + "logps/chosen": -694.9319458007812, + "logps/ref_chosen": -49.4204216003418, + "logps/ref_rejected": -80.62731170654297, + "logps/rejected": -1174.2874755859375, + "loss": 1.0873, + "margin_dpo/margin_mean": 448.14862060546875, + "margin_dpo/margin_std": 625.08984375, + "step": 595 + }, + { + "KL/chosen_KL_mean": -755.9296875, + "KL/mean": -896.432373046875, + "KL/rejected_KL_mean": -1036.93505859375, + "KL/std": 544.8298950195312, + "epoch": 0.8751835535976505, + "fcm_dpo/beta": 0.000945114647038281, + "fcm_dpo/delta": 0.1361963152885437, + "fcm_dpo/margin": 281.0052795410156, + "fcm_dpo/q_t": 0.4416961669921875, + "grad_norm": 70.67866516113281, + "learning_rate": 2.396839494982103e-08, + "logits/chosen": -1.043994665145874, + "logits/rejected": -1.005753517150879, + "logps/chosen": -815.7213745117188, + "logps/ref_chosen": -59.791683197021484, + "logps/ref_rejected": -80.09111785888672, + "logps/rejected": -1117.026123046875, + "loss": 1.2272, + "margin_dpo/margin_mean": 281.00531005859375, + "margin_dpo/margin_std": 656.0474853515625, + "step": 596 + }, + { + "KL/chosen_KL_mean": -699.6539306640625, + "KL/mean": -960.3472290039062, + "KL/rejected_KL_mean": -1221.04052734375, + "KL/std": 631.4920043945312, + "epoch": 0.8766519823788547, + "fcm_dpo/beta": 0.0009366714511997998, + "fcm_dpo/delta": -0.09220761060714722, + "fcm_dpo/margin": 521.3866577148438, + "fcm_dpo/q_t": 0.39339327812194824, + "grad_norm": 30.50818634033203, + "learning_rate": 2.3423053240837514e-08, + "logits/chosen": -0.9620223045349121, + "logits/rejected": -1.0084235668182373, + "logps/chosen": -756.9146728515625, + "logps/ref_chosen": -57.26078796386719, + "logps/ref_rejected": -100.6937255859375, + "logps/rejected": -1321.7342529296875, + "loss": 1.0648, + "margin_dpo/margin_mean": 521.3866577148438, + "margin_dpo/margin_std": 711.2468872070312, + "step": 597 + }, + { + "KL/chosen_KL_mean": -694.2019653320312, + "KL/mean": -900.156494140625, + "KL/rejected_KL_mean": -1106.111083984375, + "KL/std": 543.1419677734375, + "epoch": 0.8781204111600588, + "fcm_dpo/beta": 0.0009368107421323657, + "fcm_dpo/delta": 0.013848438858985901, + "fcm_dpo/margin": 411.9090881347656, + "fcm_dpo/q_t": 0.41072171926498413, + "grad_norm": 36.10586166381836, + "learning_rate": 2.2883682045119062e-08, + "logits/chosen": -1.0843559503555298, + "logits/rejected": -1.0926883220672607, + "logps/chosen": -746.720458984375, + "logps/ref_chosen": -52.51850509643555, + "logps/ref_rejected": -89.44385528564453, + "logps/rejected": -1195.554931640625, + "loss": 1.1177, + "margin_dpo/margin_mean": 411.9090881347656, + "margin_dpo/margin_std": 603.3158569335938, + "step": 598 + }, + { + "KL/chosen_KL_mean": -711.0474853515625, + "KL/mean": -883.8363037109375, + "KL/rejected_KL_mean": -1056.625244140625, + "KL/std": 532.8595581054688, + "epoch": 0.8795888399412628, + "fcm_dpo/beta": 0.0009362338460050523, + "fcm_dpo/delta": -0.048169832676649094, + "fcm_dpo/margin": 345.5777587890625, + "fcm_dpo/q_t": 0.4225807189941406, + "grad_norm": 32.4377326965332, + "learning_rate": 2.2350295575598367e-08, + "logits/chosen": -0.9888404011726379, + "logits/rejected": -0.994670569896698, + "logps/chosen": -760.85009765625, + "logps/ref_chosen": -49.802677154541016, + "logps/ref_rejected": -82.978515625, + "logps/rejected": -1139.603759765625, + "loss": 1.1487, + "margin_dpo/margin_mean": 345.5777587890625, + "margin_dpo/margin_std": 510.5614318847656, + "step": 599 + }, + { + "KL/chosen_KL_mean": -777.1749267578125, + "KL/mean": -945.5758056640625, + "KL/rejected_KL_mean": -1113.9766845703125, + "KL/std": 559.109619140625, + "epoch": 0.8810572687224669, + "fcm_dpo/beta": 0.000941394129768014, + "fcm_dpo/delta": 0.0843147411942482, + "fcm_dpo/margin": 336.8017272949219, + "fcm_dpo/q_t": 0.42964982986450195, + "grad_norm": 36.367210388183594, + "learning_rate": 2.1822907887504932e-08, + "logits/chosen": -1.1235439777374268, + "logits/rejected": -1.1162173748016357, + "logps/chosen": -843.6098022460938, + "logps/ref_chosen": -66.43487548828125, + "logps/ref_rejected": -85.45649719238281, + "logps/rejected": -1199.43310546875, + "loss": 1.1856, + "margin_dpo/margin_mean": 336.8017272949219, + "margin_dpo/margin_std": 669.5869140625, + "step": 600 + }, + { + "KL/chosen_KL_mean": -782.239501953125, + "KL/mean": -993.9614868164062, + "KL/rejected_KL_mean": -1205.683349609375, + "KL/std": 575.0701904296875, + "epoch": 0.882525697503671, + "fcm_dpo/beta": 0.0009437098633497953, + "fcm_dpo/delta": 0.00038953591138124466, + "fcm_dpo/margin": 423.4439697265625, + "fcm_dpo/q_t": 0.4073420763015747, + "grad_norm": 30.614376068115234, + "learning_rate": 2.1301532877994742e-08, + "logits/chosen": -1.0543596744537354, + "logits/rejected": -1.0704107284545898, + "logps/chosen": -841.373046875, + "logps/ref_chosen": -59.13361358642578, + "logps/ref_rejected": -94.69093322753906, + "logps/rejected": -1300.3743896484375, + "loss": 1.0983, + "margin_dpo/margin_mean": 423.4439697265625, + "margin_dpo/margin_std": 586.927001953125, + "step": 601 + }, + { + "KL/chosen_KL_mean": -553.6896362304688, + "KL/mean": -824.164794921875, + "KL/rejected_KL_mean": -1094.64013671875, + "KL/std": 556.9711303710938, + "epoch": 0.8839941262848752, + "fcm_dpo/beta": 0.0009391128551214933, + "fcm_dpo/delta": -0.11084318161010742, + "fcm_dpo/margin": 540.950439453125, + "fcm_dpo/q_t": 0.3849470615386963, + "grad_norm": 61.494728088378906, + "learning_rate": 2.0786184285784298e-08, + "logits/chosen": -1.071298360824585, + "logits/rejected": -1.1071069240570068, + "logps/chosen": -602.2831420898438, + "logps/ref_chosen": -48.59352111816406, + "logps/ref_rejected": -87.6685562133789, + "logps/rejected": -1182.30859375, + "loss": 1.014, + "margin_dpo/margin_mean": 540.950439453125, + "margin_dpo/margin_std": 597.431396484375, + "step": 602 + }, + { + "KL/chosen_KL_mean": -677.567138671875, + "KL/mean": -920.722900390625, + "KL/rejected_KL_mean": -1163.8787841796875, + "KL/std": 601.8591918945312, + "epoch": 0.8854625550660793, + "fcm_dpo/beta": 0.0009299052762798965, + "fcm_dpo/delta": -0.05344250053167343, + "fcm_dpo/margin": 486.3115234375, + "fcm_dpo/q_t": 0.40060174465179443, + "grad_norm": 30.66020393371582, + "learning_rate": 2.0276875690788204e-08, + "logits/chosen": -1.1096835136413574, + "logits/rejected": -1.0989437103271484, + "logps/chosen": -747.9818115234375, + "logps/ref_chosen": -70.41461944580078, + "logps/ref_rejected": -100.32559967041016, + "logps/rejected": -1264.204345703125, + "loss": 1.0749, + "margin_dpo/margin_mean": 486.3115234375, + "margin_dpo/margin_std": 690.6925048828125, + "step": 603 + }, + { + "KL/chosen_KL_mean": -667.291015625, + "KL/mean": -937.2022705078125, + "KL/rejected_KL_mean": -1207.113525390625, + "KL/std": 592.4906616210938, + "epoch": 0.8869309838472834, + "fcm_dpo/beta": 0.0009219404309988022, + "fcm_dpo/delta": -0.1001720055937767, + "fcm_dpo/margin": 539.822509765625, + "fcm_dpo/q_t": 0.3912475109100342, + "grad_norm": 30.202251434326172, + "learning_rate": 1.977362051376158e-08, + "logits/chosen": -1.0218074321746826, + "logits/rejected": -1.0635687112808228, + "logps/chosen": -713.7491455078125, + "logps/ref_chosen": -46.45808029174805, + "logps/ref_rejected": -91.8544921875, + "logps/rejected": -1298.968017578125, + "loss": 1.0509, + "margin_dpo/margin_mean": 539.822509765625, + "margin_dpo/margin_std": 732.5137939453125, + "step": 604 + }, + { + "KL/chosen_KL_mean": -696.3240966796875, + "KL/mean": -882.376708984375, + "KL/rejected_KL_mean": -1068.42919921875, + "KL/std": 532.2725219726562, + "epoch": 0.8883994126284875, + "fcm_dpo/beta": 0.0009225456742569804, + "fcm_dpo/delta": 0.05772203952074051, + "fcm_dpo/margin": 372.1051025390625, + "fcm_dpo/q_t": 0.4235140085220337, + "grad_norm": 36.44785690307617, + "learning_rate": 1.9276432015946446e-08, + "logits/chosen": -1.0042846202850342, + "logits/rejected": -1.0125752687454224, + "logps/chosen": -762.573486328125, + "logps/ref_chosen": -66.24933624267578, + "logps/ref_rejected": -102.30496978759766, + "logps/rejected": -1170.734130859375, + "loss": 1.148, + "margin_dpo/margin_mean": 372.1051025390625, + "margin_dpo/margin_std": 632.8191528320312, + "step": 605 + }, + { + "KL/chosen_KL_mean": -676.556396484375, + "KL/mean": -889.0665283203125, + "KL/rejected_KL_mean": -1101.5765380859375, + "KL/std": 537.904296875, + "epoch": 0.8898678414096917, + "fcm_dpo/beta": 0.0009270627633668482, + "fcm_dpo/delta": 0.005940597504377365, + "fcm_dpo/margin": 425.02008056640625, + "fcm_dpo/q_t": 0.4101853370666504, + "grad_norm": 31.26183319091797, + "learning_rate": 1.8785323298722093e-08, + "logits/chosen": -1.0010316371917725, + "logits/rejected": -1.0146284103393555, + "logps/chosen": -731.3755493164062, + "logps/ref_chosen": -54.819122314453125, + "logps/ref_rejected": -98.37146759033203, + "logps/rejected": -1199.947998046875, + "loss": 1.0951, + "margin_dpo/margin_mean": 425.02008056640625, + "margin_dpo/margin_std": 566.8992919921875, + "step": 606 + }, + { + "KL/chosen_KL_mean": -745.302978515625, + "KL/mean": -913.755615234375, + "KL/rejected_KL_mean": -1082.208251953125, + "KL/std": 564.3204345703125, + "epoch": 0.8913362701908958, + "fcm_dpo/beta": 0.0009324230486527085, + "fcm_dpo/delta": 0.08710216730833054, + "fcm_dpo/margin": 336.9051818847656, + "fcm_dpo/q_t": 0.4282468557357788, + "grad_norm": 29.58710479736328, + "learning_rate": 1.8300307303259904e-08, + "logits/chosen": -0.9958839416503906, + "logits/rejected": -0.9777606129646301, + "logps/chosen": -803.3870239257812, + "logps/ref_chosen": -58.08403778076172, + "logps/ref_rejected": -79.777099609375, + "logps/rejected": -1161.9853515625, + "loss": 1.1734, + "margin_dpo/margin_mean": 336.9051513671875, + "margin_dpo/margin_std": 613.3471069335938, + "step": 607 + }, + { + "KL/chosen_KL_mean": -618.9171142578125, + "KL/mean": -831.6253662109375, + "KL/rejected_KL_mean": -1044.3336181640625, + "KL/std": 493.6427001953125, + "epoch": 0.8928046989720999, + "fcm_dpo/beta": 0.000934786512516439, + "fcm_dpo/delta": 0.002314794808626175, + "fcm_dpo/margin": 425.41650390625, + "fcm_dpo/q_t": 0.4072082042694092, + "grad_norm": 28.977453231811523, + "learning_rate": 1.7821396810182437e-08, + "logits/chosen": -1.0519757270812988, + "logits/rejected": -1.0653681755065918, + "logps/chosen": -676.367919921875, + "logps/ref_chosen": -57.450836181640625, + "logps/ref_rejected": -94.77339172363281, + "logps/rejected": -1139.10693359375, + "loss": 1.0788, + "margin_dpo/margin_mean": 425.41650390625, + "margin_dpo/margin_std": 498.175048828125, + "step": 608 + }, + { + "KL/chosen_KL_mean": -661.7661743164062, + "KL/mean": -925.5433349609375, + "KL/rejected_KL_mean": -1189.320556640625, + "KL/std": 666.0048828125, + "epoch": 0.8942731277533039, + "fcm_dpo/beta": 0.0009277096251025796, + "fcm_dpo/delta": -0.09163244068622589, + "fcm_dpo/margin": 527.5543823242188, + "fcm_dpo/q_t": 0.3955400884151459, + "grad_norm": 30.367385864257812, + "learning_rate": 1.7348604439226617e-08, + "logits/chosen": -1.1196882724761963, + "logits/rejected": -1.1408710479736328, + "logps/chosen": -720.571533203125, + "logps/ref_chosen": -58.805355072021484, + "logps/ref_rejected": -88.81600952148438, + "logps/rejected": -1278.136474609375, + "loss": 1.0618, + "margin_dpo/margin_mean": 527.5543823242188, + "margin_dpo/margin_std": 781.7075805664062, + "step": 609 + }, + { + "KL/chosen_KL_mean": -647.3359375, + "KL/mean": -821.218505859375, + "KL/rejected_KL_mean": -995.10107421875, + "KL/std": 509.16314697265625, + "epoch": 0.895741556534508, + "fcm_dpo/beta": 0.0009309325832873583, + "fcm_dpo/delta": 0.07747611403465271, + "fcm_dpo/margin": 347.76507568359375, + "fcm_dpo/q_t": 0.42488405108451843, + "grad_norm": 41.63359451293945, + "learning_rate": 1.6881942648911074e-08, + "logits/chosen": -1.0002460479736328, + "logits/rejected": -0.9726261496543884, + "logps/chosen": -713.031005859375, + "logps/ref_chosen": -65.69503784179688, + "logps/ref_rejected": -83.40538787841797, + "logps/rejected": -1078.5064697265625, + "loss": 1.1582, + "margin_dpo/margin_mean": 347.76507568359375, + "margin_dpo/margin_std": 584.4713745117188, + "step": 610 + }, + { + "KL/chosen_KL_mean": -680.7236328125, + "KL/mean": -954.4203491210938, + "KL/rejected_KL_mean": -1228.1170654296875, + "KL/std": 674.031494140625, + "epoch": 0.8972099853157122, + "fcm_dpo/beta": 0.0009249848080798984, + "fcm_dpo/delta": -0.1090783104300499, + "fcm_dpo/margin": 547.3934326171875, + "fcm_dpo/q_t": 0.3915501832962036, + "grad_norm": 40.06733703613281, + "learning_rate": 1.6421423736208e-08, + "logits/chosen": -1.0733530521392822, + "logits/rejected": -1.1131242513656616, + "logps/chosen": -733.3231201171875, + "logps/ref_chosen": -52.59946823120117, + "logps/ref_rejected": -86.33099365234375, + "logps/rejected": -1314.447998046875, + "loss": 1.057, + "margin_dpo/margin_mean": 547.3934326171875, + "margin_dpo/margin_std": 783.890869140625, + "step": 611 + }, + { + "KL/chosen_KL_mean": -730.0038452148438, + "KL/mean": -944.6229248046875, + "KL/rejected_KL_mean": -1159.2420654296875, + "KL/std": 537.7095336914062, + "epoch": 0.8986784140969163, + "fcm_dpo/beta": 0.0009237851481884718, + "fcm_dpo/delta": 0.003468703478574753, + "fcm_dpo/margin": 429.2381896972656, + "fcm_dpo/q_t": 0.40873080492019653, + "grad_norm": 30.787771224975586, + "learning_rate": 1.5967059836219042e-08, + "logits/chosen": -1.054150104522705, + "logits/rejected": -1.0440592765808105, + "logps/chosen": -789.3275756835938, + "logps/ref_chosen": -59.32372283935547, + "logps/ref_rejected": -88.31239318847656, + "logps/rejected": -1247.554443359375, + "loss": 1.0915, + "margin_dpo/margin_mean": 429.2381591796875, + "margin_dpo/margin_std": 562.5599975585938, + "step": 612 + }, + { + "KL/chosen_KL_mean": -648.699951171875, + "KL/mean": -902.3504028320312, + "KL/rejected_KL_mean": -1156.0009765625, + "KL/std": 597.5882568359375, + "epoch": 0.9001468428781204, + "fcm_dpo/beta": 0.0009168373653665185, + "fcm_dpo/delta": -0.066818006336689, + "fcm_dpo/margin": 507.301025390625, + "fcm_dpo/q_t": 0.39431166648864746, + "grad_norm": 34.06965255737305, + "learning_rate": 1.551886292185553e-08, + "logits/chosen": -1.0495436191558838, + "logits/rejected": -1.0977256298065186, + "logps/chosen": -708.4298706054688, + "logps/ref_chosen": -59.72996520996094, + "logps/ref_rejected": -105.10752868652344, + "logps/rejected": -1261.1083984375, + "loss": 1.048, + "margin_dpo/margin_mean": 507.301025390625, + "margin_dpo/margin_std": 619.2185668945312, + "step": 613 + }, + { + "KL/chosen_KL_mean": -726.5540771484375, + "KL/mean": -973.306640625, + "KL/rejected_KL_mean": -1220.0592041015625, + "KL/std": 595.4200439453125, + "epoch": 0.9016152716593245, + "fcm_dpo/beta": 0.0009136309381574392, + "fcm_dpo/delta": -0.05203912407159805, + "fcm_dpo/margin": 493.5052185058594, + "fcm_dpo/q_t": 0.3989720642566681, + "grad_norm": 43.53131866455078, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": -1.0377488136291504, + "logits/rejected": -1.1064826250076294, + "logps/chosen": -779.4930419921875, + "logps/ref_chosen": -52.93898010253906, + "logps/ref_rejected": -104.67938232421875, + "logps/rejected": -1324.738525390625, + "loss": 1.0802, + "margin_dpo/margin_mean": 493.50518798828125, + "margin_dpo/margin_std": 707.6263427734375, + "step": 614 + }, + { + "KL/chosen_KL_mean": -656.21240234375, + "KL/mean": -860.7072143554688, + "KL/rejected_KL_mean": -1065.2021484375, + "KL/std": 597.1307373046875, + "epoch": 0.9030837004405287, + "fcm_dpo/beta": 0.0009152768179774284, + "fcm_dpo/delta": 0.025946544483304024, + "fcm_dpo/margin": 408.9896545410156, + "fcm_dpo/q_t": 0.4139822721481323, + "grad_norm": 33.06850814819336, + "learning_rate": 1.4641017128809801e-08, + "logits/chosen": -0.9963966608047485, + "logits/rejected": -1.0074797868728638, + "logps/chosen": -722.0296630859375, + "logps/ref_chosen": -65.81727600097656, + "logps/ref_rejected": -95.17749786376953, + "logps/rejected": -1160.379638671875, + "loss": 1.1254, + "margin_dpo/margin_mean": 408.9896545410156, + "margin_dpo/margin_std": 637.387451171875, + "step": 615 + }, + { + "KL/chosen_KL_mean": -777.4338989257812, + "KL/mean": -951.5509033203125, + "KL/rejected_KL_mean": -1125.66796875, + "KL/std": 516.37451171875, + "epoch": 0.9045521292217328, + "fcm_dpo/beta": 0.0009200784843415022, + "fcm_dpo/delta": 0.08084507286548615, + "fcm_dpo/margin": 348.23406982421875, + "fcm_dpo/q_t": 0.4268506169319153, + "grad_norm": 27.89613151550293, + "learning_rate": 1.4211391382180637e-08, + "logits/chosen": -1.0936863422393799, + "logits/rejected": -1.075373649597168, + "logps/chosen": -842.5667724609375, + "logps/ref_chosen": -65.13285827636719, + "logps/ref_rejected": -74.70050048828125, + "logps/rejected": -1200.3685302734375, + "loss": 1.1618, + "margin_dpo/margin_mean": 348.2341003417969, + "margin_dpo/margin_std": 590.9950561523438, + "step": 616 + }, + { + "KL/chosen_KL_mean": -742.44384765625, + "KL/mean": -881.6417846679688, + "KL/rejected_KL_mean": -1020.8397216796875, + "KL/std": 495.73095703125, + "epoch": 0.9060205580029369, + "fcm_dpo/beta": 0.0009322729893028736, + "fcm_dpo/delta": 0.1421297788619995, + "fcm_dpo/margin": 278.3958740234375, + "fcm_dpo/q_t": 0.4408647119998932, + "grad_norm": 45.14849853515625, + "learning_rate": 1.378797888467345e-08, + "logits/chosen": -0.9931391477584839, + "logits/rejected": -0.9588996171951294, + "logps/chosen": -805.4493408203125, + "logps/ref_chosen": -63.005550384521484, + "logps/ref_rejected": -64.234130859375, + "logps/rejected": -1085.0738525390625, + "loss": 1.2146, + "margin_dpo/margin_mean": 278.3958740234375, + "margin_dpo/margin_std": 587.977294921875, + "step": 617 + }, + { + "KL/chosen_KL_mean": -761.5391845703125, + "KL/mean": -1049.184814453125, + "KL/rejected_KL_mean": -1336.83056640625, + "KL/std": 683.9238891601562, + "epoch": 0.9074889867841409, + "fcm_dpo/beta": 0.0009282865794375539, + "fcm_dpo/delta": -0.1377088725566864, + "fcm_dpo/margin": 575.2913818359375, + "fcm_dpo/q_t": 0.3874184489250183, + "grad_norm": 37.7491455078125, + "learning_rate": 1.3370790793601371e-08, + "logits/chosen": -1.0628851652145386, + "logits/rejected": -1.1012842655181885, + "logps/chosen": -828.6405639648438, + "logps/ref_chosen": -67.10134887695312, + "logps/ref_rejected": -92.15340423583984, + "logps/rejected": -1428.98388671875, + "loss": 1.0682, + "margin_dpo/margin_mean": 575.2913818359375, + "margin_dpo/margin_std": 880.9173583984375, + "step": 618 + }, + { + "KL/chosen_KL_mean": -778.9534301757812, + "KL/mean": -977.70556640625, + "KL/rejected_KL_mean": -1176.457763671875, + "KL/std": 610.49609375, + "epoch": 0.908957415565345, + "fcm_dpo/beta": 0.0009244297398254275, + "fcm_dpo/delta": 0.03313371166586876, + "fcm_dpo/margin": 397.50433349609375, + "fcm_dpo/q_t": 0.4229835271835327, + "grad_norm": 41.30840301513672, + "learning_rate": 1.2959838102258535e-08, + "logits/chosen": -1.0145196914672852, + "logits/rejected": -1.0216684341430664, + "logps/chosen": -834.931640625, + "logps/ref_chosen": -55.978233337402344, + "logps/ref_rejected": -93.1854019165039, + "logps/rejected": -1269.64306640625, + "loss": 1.1829, + "margin_dpo/margin_mean": 397.5043029785156, + "margin_dpo/margin_std": 825.1723022460938, + "step": 619 + }, + { + "KL/chosen_KL_mean": -700.730224609375, + "KL/mean": -897.155517578125, + "KL/rejected_KL_mean": -1093.5806884765625, + "KL/std": 533.931640625, + "epoch": 0.9104258443465492, + "fcm_dpo/beta": 0.0009257015772163868, + "fcm_dpo/delta": 0.03692508488893509, + "fcm_dpo/margin": 392.8504638671875, + "fcm_dpo/q_t": 0.4179452955722809, + "grad_norm": 41.25801467895508, + "learning_rate": 1.2555131639630567e-08, + "logits/chosen": -1.0765249729156494, + "logits/rejected": -1.0782887935638428, + "logps/chosen": -760.5277099609375, + "logps/ref_chosen": -59.79750061035156, + "logps/ref_rejected": -78.41075134277344, + "logps/rejected": -1171.991455078125, + "loss": 1.1321, + "margin_dpo/margin_mean": 392.8504943847656, + "margin_dpo/margin_std": 619.072998046875, + "step": 620 + }, + { + "KL/chosen_KL_mean": -700.159912109375, + "KL/mean": -985.0526123046875, + "KL/rejected_KL_mean": -1269.9453125, + "KL/std": 644.7921142578125, + "epoch": 0.9118942731277533, + "fcm_dpo/beta": 0.000923096900805831, + "fcm_dpo/delta": -0.12948347628116608, + "fcm_dpo/margin": 569.7854614257812, + "fcm_dpo/q_t": 0.3818792700767517, + "grad_norm": 36.94768142700195, + "learning_rate": 1.2156682070109086e-08, + "logits/chosen": -1.092390775680542, + "logits/rejected": -1.13364839553833, + "logps/chosen": -754.0936279296875, + "logps/ref_chosen": -53.93375778198242, + "logps/ref_rejected": -88.36951446533203, + "logps/rejected": -1358.31494140625, + "loss": 1.027, + "margin_dpo/margin_mean": 569.7854614257812, + "margin_dpo/margin_std": 705.4525146484375, + "step": 621 + }, + { + "KL/chosen_KL_mean": -697.857421875, + "KL/mean": -898.1604614257812, + "KL/rejected_KL_mean": -1098.463623046875, + "KL/std": 515.55908203125, + "epoch": 0.9133627019089574, + "fcm_dpo/beta": 0.0009166207164525986, + "fcm_dpo/delta": 0.0332704596221447, + "fcm_dpo/margin": 400.6061096191406, + "fcm_dpo/q_t": 0.4171668291091919, + "grad_norm": 37.14033889770508, + "learning_rate": 1.1764499893210878e-08, + "logits/chosen": -0.9720423221588135, + "logits/rejected": -0.9517272710800171, + "logps/chosen": -758.1432495117188, + "logps/ref_chosen": -60.28582000732422, + "logps/ref_rejected": -85.51873779296875, + "logps/rejected": -1183.9822998046875, + "loss": 1.1303, + "margin_dpo/margin_mean": 400.6060791015625, + "margin_dpo/margin_std": 636.8636474609375, + "step": 622 + }, + { + "KL/chosen_KL_mean": -729.516845703125, + "KL/mean": -890.4384765625, + "KL/rejected_KL_mean": -1051.3599853515625, + "KL/std": 516.193359375, + "epoch": 0.9148311306901615, + "fcm_dpo/beta": 0.0009268617141060531, + "fcm_dpo/delta": 0.10320872068405151, + "fcm_dpo/margin": 321.8431701660156, + "fcm_dpo/q_t": 0.43382978439331055, + "grad_norm": 35.06966018676758, + "learning_rate": 1.1378595443300998e-08, + "logits/chosen": -1.096697211265564, + "logits/rejected": -1.092256784439087, + "logps/chosen": -793.673828125, + "logps/ref_chosen": -64.1569595336914, + "logps/ref_rejected": -85.08304595947266, + "logps/rejected": -1136.443115234375, + "loss": 1.1852, + "margin_dpo/margin_mean": 321.8431396484375, + "margin_dpo/margin_std": 620.9134521484375, + "step": 623 + }, + { + "KL/chosen_KL_mean": -732.138916015625, + "KL/mean": -979.6350708007812, + "KL/rejected_KL_mean": -1227.131103515625, + "KL/std": 541.4876708984375, + "epoch": 0.9162995594713657, + "fcm_dpo/beta": 0.0009254544274881482, + "fcm_dpo/delta": -0.05943404883146286, + "fcm_dpo/margin": 494.9923400878906, + "fcm_dpo/q_t": 0.39315974712371826, + "grad_norm": 33.594234466552734, + "learning_rate": 1.0998978889320582e-08, + "logits/chosen": -1.132512092590332, + "logits/rejected": -1.1252646446228027, + "logps/chosen": -804.0575561523438, + "logps/ref_chosen": -71.91862487792969, + "logps/ref_rejected": -97.13203430175781, + "logps/rejected": -1324.26318359375, + "loss": 1.0492, + "margin_dpo/margin_mean": 494.9923400878906, + "margin_dpo/margin_std": 585.3521118164062, + "step": 624 + }, + { + "KL/chosen_KL_mean": -680.2445068359375, + "KL/mean": -972.3754272460938, + "KL/rejected_KL_mean": -1264.50634765625, + "KL/std": 626.7379150390625, + "epoch": 0.9177679882525698, + "fcm_dpo/beta": 0.0009159696055576205, + "fcm_dpo/delta": -0.13877010345458984, + "fcm_dpo/margin": 584.261962890625, + "fcm_dpo/q_t": 0.3786303400993347, + "grad_norm": 48.92192459106445, + "learning_rate": 1.0625660234518913e-08, + "logits/chosen": -1.0401864051818848, + "logits/rejected": -1.0676482915878296, + "logps/chosen": -738.5865478515625, + "logps/ref_chosen": -58.342071533203125, + "logps/ref_rejected": -86.09038543701172, + "logps/rejected": -1350.5968017578125, + "loss": 0.9865, + "margin_dpo/margin_mean": 584.261962890625, + "margin_dpo/margin_std": 589.0737915039062, + "step": 625 + }, + { + "KL/chosen_KL_mean": -864.771484375, + "KL/mean": -1021.89208984375, + "KL/rejected_KL_mean": -1179.0126953125, + "KL/std": 683.1890869140625, + "epoch": 0.9192364170337739, + "fcm_dpo/beta": 0.0009178274776786566, + "fcm_dpo/delta": 0.1132790595293045, + "fcm_dpo/margin": 314.2411193847656, + "fcm_dpo/q_t": 0.4319329857826233, + "grad_norm": 38.202056884765625, + "learning_rate": 1.0258649316189721e-08, + "logits/chosen": -1.0161819458007812, + "logits/rejected": -1.007856011390686, + "logps/chosen": -939.8840942382812, + "logps/ref_chosen": -75.11260986328125, + "logps/ref_rejected": -99.188720703125, + "logps/rejected": -1278.201416015625, + "loss": 1.2125, + "margin_dpo/margin_mean": 314.2411193847656, + "margin_dpo/margin_std": 675.1224365234375, + "step": 626 + }, + { + "KL/chosen_KL_mean": -591.8564453125, + "KL/mean": -881.7455444335938, + "KL/rejected_KL_mean": -1171.6346435546875, + "KL/std": 698.69775390625, + "epoch": 0.920704845814978, + "fcm_dpo/beta": 0.0009149962570518255, + "fcm_dpo/delta": -0.13404129445552826, + "fcm_dpo/margin": 579.7781372070312, + "fcm_dpo/q_t": 0.3895081877708435, + "grad_norm": 31.1433162689209, + "learning_rate": 9.897955805412e-09, + "logits/chosen": -0.9388647079467773, + "logits/rejected": -1.0119527578353882, + "logps/chosen": -639.599609375, + "logps/ref_chosen": -47.74314880371094, + "logps/ref_rejected": -106.75448608398438, + "logps/rejected": -1278.38916015625, + "loss": 1.0424, + "margin_dpo/margin_mean": 579.7781372070312, + "margin_dpo/margin_std": 799.517578125, + "step": 627 + }, + { + "KL/chosen_KL_mean": -759.9505615234375, + "KL/mean": -988.3452758789062, + "KL/rejected_KL_mean": -1216.739990234375, + "KL/std": 587.220458984375, + "epoch": 0.922173274596182, + "fcm_dpo/beta": 0.0009073036490008235, + "fcm_dpo/delta": -0.014781368896365166, + "fcm_dpo/margin": 456.78936767578125, + "fcm_dpo/q_t": 0.40726301074028015, + "grad_norm": 34.24974060058594, + "learning_rate": 9.543589206795238e-09, + "logits/chosen": -1.096536636352539, + "logits/rejected": -1.1101562976837158, + "logps/chosen": -820.1334838867188, + "logps/ref_chosen": -60.182945251464844, + "logps/ref_rejected": -101.55467224121094, + "logps/rejected": -1318.294677734375, + "loss": 1.1024, + "margin_dpo/margin_mean": 456.78936767578125, + "margin_dpo/margin_std": 680.5447998046875, + "step": 628 + }, + { + "KL/chosen_KL_mean": -751.41064453125, + "KL/mean": -967.8169555664062, + "KL/rejected_KL_mean": -1184.2232666015625, + "KL/std": 572.2176513671875, + "epoch": 0.9236417033773862, + "fcm_dpo/beta": 0.0009084055200219154, + "fcm_dpo/delta": 0.006959038320928812, + "fcm_dpo/margin": 432.81256103515625, + "fcm_dpo/q_t": 0.40856897830963135, + "grad_norm": 38.99527359008789, + "learning_rate": 9.19555885822887e-09, + "logits/chosen": -1.0798592567443848, + "logits/rejected": -1.0886130332946777, + "logps/chosen": -815.6241455078125, + "logps/ref_chosen": -64.21354675292969, + "logps/ref_rejected": -91.65367126464844, + "logps/rejected": -1275.876953125, + "loss": 1.0948, + "margin_dpo/margin_mean": 432.81256103515625, + "margin_dpo/margin_std": 571.304443359375, + "step": 629 + }, + { + "KL/chosen_KL_mean": -677.60888671875, + "KL/mean": -813.6373291015625, + "KL/rejected_KL_mean": -949.665771484375, + "KL/std": 625.6536254882812, + "epoch": 0.9251101321585903, + "fcm_dpo/beta": 0.0009094603592529893, + "fcm_dpo/delta": 0.0273385439068079, + "fcm_dpo/margin": 272.05682373046875, + "fcm_dpo/q_t": 0.4511290192604065, + "grad_norm": 44.24210739135742, + "learning_rate": 8.85387393063622e-09, + "logits/chosen": -1.0548908710479736, + "logits/rejected": -1.0271886587142944, + "logps/chosen": -736.89990234375, + "logps/ref_chosen": -59.29100036621094, + "logps/ref_rejected": -83.59829711914062, + "logps/rejected": -1033.2640380859375, + "loss": 1.2546, + "margin_dpo/margin_mean": 272.0568542480469, + "margin_dpo/margin_std": 752.2532958984375, + "step": 630 + }, + { + "KL/chosen_KL_mean": -818.7833862304688, + "KL/mean": -1021.9134521484375, + "KL/rejected_KL_mean": -1225.04345703125, + "KL/std": 593.9693603515625, + "epoch": 0.9265785609397944, + "fcm_dpo/beta": 0.0009135643485933542, + "fcm_dpo/delta": 0.029337339103221893, + "fcm_dpo/margin": 406.260009765625, + "fcm_dpo/q_t": 0.4151837229728699, + "grad_norm": 36.602779388427734, + "learning_rate": 8.518543427732949e-09, + "logits/chosen": -1.141347885131836, + "logits/rejected": -1.1473594903945923, + "logps/chosen": -878.2369995117188, + "logps/ref_chosen": -59.45360565185547, + "logps/ref_rejected": -80.95156860351562, + "logps/rejected": -1305.9949951171875, + "loss": 1.14, + "margin_dpo/margin_mean": 406.2600402832031, + "margin_dpo/margin_std": 679.403564453125, + "step": 631 + }, + { + "KL/chosen_KL_mean": -730.484130859375, + "KL/mean": -936.6009521484375, + "KL/rejected_KL_mean": -1142.7177734375, + "KL/std": 546.6138305664062, + "epoch": 0.9280469897209985, + "fcm_dpo/beta": 0.0009125604410655797, + "fcm_dpo/delta": 0.024038948118686676, + "fcm_dpo/margin": 412.233642578125, + "fcm_dpo/q_t": 0.4137340188026428, + "grad_norm": 39.51885986328125, + "learning_rate": 8.189576185789637e-09, + "logits/chosen": -1.1271458864212036, + "logits/rejected": -1.126634955406189, + "logps/chosen": -791.835693359375, + "logps/ref_chosen": -61.35155487060547, + "logps/ref_rejected": -86.16017150878906, + "logps/rejected": -1228.8779296875, + "loss": 1.1307, + "margin_dpo/margin_mean": 412.233642578125, + "margin_dpo/margin_std": 653.526611328125, + "step": 632 + }, + { + "KL/chosen_KL_mean": -793.9225463867188, + "KL/mean": -951.2642822265625, + "KL/rejected_KL_mean": -1108.6060791015625, + "KL/std": 532.9287109375, + "epoch": 0.9295154185022027, + "fcm_dpo/beta": 0.0009164665825664997, + "fcm_dpo/delta": 0.019403798505663872, + "fcm_dpo/margin": 314.6834716796875, + "fcm_dpo/q_t": 0.4337080717086792, + "grad_norm": 44.8255615234375, + "learning_rate": 7.866980873399015e-09, + "logits/chosen": -1.1348984241485596, + "logits/rejected": -1.147522211074829, + "logps/chosen": -851.20068359375, + "logps/ref_chosen": -57.27816390991211, + "logps/ref_rejected": -91.58395385742188, + "logps/rejected": -1200.1900634765625, + "loss": 1.1933, + "margin_dpo/margin_mean": 314.6835021972656, + "margin_dpo/margin_std": 608.6378784179688, + "step": 633 + }, + { + "KL/chosen_KL_mean": -881.3724365234375, + "KL/mean": -1016.420654296875, + "KL/rejected_KL_mean": -1151.4688720703125, + "KL/std": 623.2406005859375, + "epoch": 0.9309838472834068, + "fcm_dpo/beta": 0.0009277792414650321, + "fcm_dpo/delta": 0.15131084620952606, + "fcm_dpo/margin": 270.09649658203125, + "fcm_dpo/q_t": 0.4442192018032074, + "grad_norm": 36.3792724609375, + "learning_rate": 7.550765991247654e-09, + "logits/chosen": -0.985339879989624, + "logits/rejected": -0.9769416451454163, + "logps/chosen": -947.991455078125, + "logps/ref_chosen": -66.61896514892578, + "logps/ref_rejected": -107.12564849853516, + "logps/rejected": -1258.594482421875, + "loss": 1.2225, + "margin_dpo/margin_mean": 270.09649658203125, + "margin_dpo/margin_std": 597.4962768554688, + "step": 634 + }, + { + "KL/chosen_KL_mean": -775.334716796875, + "KL/mean": -957.4782104492188, + "KL/rejected_KL_mean": -1139.62158203125, + "KL/std": 646.2401733398438, + "epoch": 0.9324522760646109, + "fcm_dpo/beta": 0.0009354540961794555, + "fcm_dpo/delta": 0.060258638113737106, + "fcm_dpo/margin": 364.28692626953125, + "fcm_dpo/q_t": 0.4247502088546753, + "grad_norm": 43.95642852783203, + "learning_rate": 7.240939871891699e-09, + "logits/chosen": -1.0711841583251953, + "logits/rejected": -1.0444178581237793, + "logps/chosen": -849.290283203125, + "logps/ref_chosen": -73.95551300048828, + "logps/ref_rejected": -82.50045776367188, + "logps/rejected": -1222.1220703125, + "loss": 1.163, + "margin_dpo/margin_mean": 364.28692626953125, + "margin_dpo/margin_std": 672.04248046875, + "step": 635 + }, + { + "KL/chosen_KL_mean": -732.2950439453125, + "KL/mean": -943.854248046875, + "KL/rejected_KL_mean": -1155.413330078125, + "KL/std": 642.1853637695312, + "epoch": 0.933920704845815, + "fcm_dpo/beta": 0.0009403472067788243, + "fcm_dpo/delta": 0.0016737580299377441, + "fcm_dpo/margin": 423.1182861328125, + "fcm_dpo/q_t": 0.41292804479599, + "grad_norm": 25.051197052001953, + "learning_rate": 6.937510679537628e-09, + "logits/chosen": -0.9883960485458374, + "logits/rejected": -0.9851752519607544, + "logps/chosen": -791.9239501953125, + "logps/ref_chosen": -59.628910064697266, + "logps/ref_rejected": -81.97883605957031, + "logps/rejected": -1237.3922119140625, + "loss": 1.1096, + "margin_dpo/margin_mean": 423.1182861328125, + "margin_dpo/margin_std": 657.2255859375, + "step": 636 + }, + { + "KL/chosen_KL_mean": -731.9287719726562, + "KL/mean": -977.4234619140625, + "KL/rejected_KL_mean": -1222.9180908203125, + "KL/std": 657.0423583984375, + "epoch": 0.9353891336270191, + "fcm_dpo/beta": 0.0009306943975389004, + "fcm_dpo/delta": -0.05869361758232117, + "fcm_dpo/margin": 490.98931884765625, + "fcm_dpo/q_t": 0.39948800206184387, + "grad_norm": 32.110877990722656, + "learning_rate": 6.640486409826785e-09, + "logits/chosen": -1.1167852878570557, + "logits/rejected": -1.1629787683486938, + "logps/chosen": -781.5814208984375, + "logps/ref_chosen": -49.652687072753906, + "logps/ref_rejected": -98.40513610839844, + "logps/rejected": -1321.3232421875, + "loss": 1.0728, + "margin_dpo/margin_mean": 490.98931884765625, + "margin_dpo/margin_std": 690.1351318359375, + "step": 637 + }, + { + "KL/chosen_KL_mean": -686.4628295898438, + "KL/mean": -884.8165283203125, + "KL/rejected_KL_mean": -1083.1702880859375, + "KL/std": 587.4907836914062, + "epoch": 0.9368575624082232, + "fcm_dpo/beta": 0.0009277926292270422, + "fcm_dpo/delta": -0.08301420509815216, + "fcm_dpo/margin": 396.7074279785156, + "fcm_dpo/q_t": 0.4109494686126709, + "grad_norm": 36.82447052001953, + "learning_rate": 6.349874889624962e-09, + "logits/chosen": -0.9534268379211426, + "logits/rejected": -0.9300310611724854, + "logps/chosen": -744.6195068359375, + "logps/ref_chosen": -58.156639099121094, + "logps/ref_rejected": -79.3014907836914, + "logps/rejected": -1162.4716796875, + "loss": 1.1481, + "margin_dpo/margin_mean": 396.7074279785156, + "margin_dpo/margin_std": 668.0907592773438, + "step": 638 + }, + { + "KL/chosen_KL_mean": -987.62060546875, + "KL/mean": -1060.539306640625, + "KL/rejected_KL_mean": -1133.458251953125, + "KL/std": 582.9657592773438, + "epoch": 0.9383259911894273, + "fcm_dpo/beta": 0.0009239416103810072, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 145.8376007080078, + "fcm_dpo/q_t": 0.46903330087661743, + "grad_norm": 139.1773681640625, + "learning_rate": 6.065683776815933e-09, + "logits/chosen": -0.9747638702392578, + "logits/rejected": -0.9038654565811157, + "logps/chosen": -1059.94384765625, + "logps/ref_chosen": -72.32319641113281, + "logps/ref_rejected": -74.2749252319336, + "logps/rejected": -1207.733154296875, + "loss": 1.3733, + "margin_dpo/margin_mean": 145.83761596679688, + "margin_dpo/margin_std": 759.8252563476562, + "step": 639 + }, + { + "KL/chosen_KL_mean": -737.2076416015625, + "KL/mean": -1026.8175048828125, + "KL/rejected_KL_mean": -1316.42724609375, + "KL/std": 669.41357421875, + "epoch": 0.9397944199706314, + "fcm_dpo/beta": 0.00091610784875229, + "fcm_dpo/delta": -0.13407810032367706, + "fcm_dpo/margin": 579.2196044921875, + "fcm_dpo/q_t": 0.3861461579799652, + "grad_norm": 52.59541320800781, + "learning_rate": 5.7879205600998296e-09, + "logits/chosen": -0.9671276807785034, + "logits/rejected": -0.9929375648498535, + "logps/chosen": -793.342041015625, + "logps/ref_chosen": -56.13436508178711, + "logps/ref_rejected": -108.60014343261719, + "logps/rejected": -1425.02734375, + "loss": 1.0344, + "margin_dpo/margin_mean": 579.2196044921875, + "margin_dpo/margin_std": 785.578369140625, + "step": 640 + }, + { + "KL/chosen_KL_mean": -859.6257934570312, + "KL/mean": -1046.6640625, + "KL/rejected_KL_mean": -1233.7021484375, + "KL/std": 573.8028564453125, + "epoch": 0.9412628487518355, + "fcm_dpo/beta": 0.0009146465454250574, + "fcm_dpo/delta": 0.05887487903237343, + "fcm_dpo/margin": 374.076416015625, + "fcm_dpo/q_t": 0.42490917444229126, + "grad_norm": 37.22019958496094, + "learning_rate": 5.516592558795746e-09, + "logits/chosen": -1.0567216873168945, + "logits/rejected": -1.0576525926589966, + "logps/chosen": -924.6226806640625, + "logps/ref_chosen": -64.99689483642578, + "logps/ref_rejected": -86.99232482910156, + "logps/rejected": -1320.694580078125, + "loss": 1.1778, + "margin_dpo/margin_mean": 374.076416015625, + "margin_dpo/margin_std": 746.7939453125, + "step": 641 + }, + { + "KL/chosen_KL_mean": -806.0206298828125, + "KL/mean": -1029.8985595703125, + "KL/rejected_KL_mean": -1253.7764892578125, + "KL/std": 721.0260620117188, + "epoch": 0.9427312775330396, + "fcm_dpo/beta": 0.0009186153765767813, + "fcm_dpo/delta": -0.011798828840255737, + "fcm_dpo/margin": 447.75579833984375, + "fcm_dpo/q_t": 0.41545358300209045, + "grad_norm": 39.59940719604492, + "learning_rate": 5.251706922648868e-09, + "logits/chosen": -1.001420021057129, + "logits/rejected": -1.0281758308410645, + "logps/chosen": -871.7098999023438, + "logps/ref_chosen": -65.68924713134766, + "logps/ref_rejected": -110.24205017089844, + "logps/rejected": -1364.0185546875, + "loss": 1.1498, + "margin_dpo/margin_mean": 447.75579833984375, + "margin_dpo/margin_std": 873.0943603515625, + "step": 642 + }, + { + "KL/chosen_KL_mean": -727.7049560546875, + "KL/mean": -922.38037109375, + "KL/rejected_KL_mean": -1117.055908203125, + "KL/std": 554.1806640625, + "epoch": 0.9441997063142438, + "fcm_dpo/beta": 0.0009108081576414406, + "fcm_dpo/delta": -0.05616462975740433, + "fcm_dpo/margin": 389.350830078125, + "fcm_dpo/q_t": 0.41762280464172363, + "grad_norm": 45.18101119995117, + "learning_rate": 4.993270631642038e-09, + "logits/chosen": -1.1256649494171143, + "logits/rejected": -1.1269869804382324, + "logps/chosen": -779.6549072265625, + "logps/ref_chosen": -51.94999694824219, + "logps/ref_rejected": -87.46833801269531, + "logps/rejected": -1204.524169921875, + "loss": 1.1243, + "margin_dpo/margin_mean": 389.3507995605469, + "margin_dpo/margin_std": 536.47412109375, + "step": 643 + }, + { + "KL/chosen_KL_mean": -711.8359375, + "KL/mean": -895.3848876953125, + "KL/rejected_KL_mean": -1078.9339599609375, + "KL/std": 636.5994873046875, + "epoch": 0.9456681350954479, + "fcm_dpo/beta": 0.0009149383986368775, + "fcm_dpo/delta": 0.06524142622947693, + "fcm_dpo/margin": 367.0979919433594, + "fcm_dpo/q_t": 0.42591458559036255, + "grad_norm": 48.07936096191406, + "learning_rate": 4.741290495811873e-09, + "logits/chosen": -1.0160619020462036, + "logits/rejected": -1.021895170211792, + "logps/chosen": -770.8535766601562, + "logps/ref_chosen": -59.017662048339844, + "logps/ref_rejected": -87.13668823242188, + "logps/rejected": -1166.070556640625, + "loss": 1.1859, + "margin_dpo/margin_mean": 367.0979919433594, + "margin_dpo/margin_std": 747.0296020507812, + "step": 644 + }, + { + "KL/chosen_KL_mean": -738.4013061523438, + "KL/mean": -835.6104125976562, + "KL/rejected_KL_mean": -932.8195190429688, + "KL/std": 500.1334228515625, + "epoch": 0.947136563876652, + "fcm_dpo/beta": 0.0009230460273101926, + "fcm_dpo/delta": 0.068226657807827, + "fcm_dpo/margin": 194.41815185546875, + "fcm_dpo/q_t": 0.4604693055152893, + "grad_norm": 92.3852310180664, + "learning_rate": 4.495773155069299e-09, + "logits/chosen": -0.9825325608253479, + "logits/rejected": -0.969984769821167, + "logps/chosen": -794.27734375, + "logps/ref_chosen": -55.87602233886719, + "logps/ref_rejected": -97.78080749511719, + "logps/rejected": -1030.600341796875, + "loss": 1.3114, + "margin_dpo/margin_mean": 194.41818237304688, + "margin_dpo/margin_std": 666.4583740234375, + "step": 645 + }, + { + "KL/chosen_KL_mean": -734.58056640625, + "KL/mean": -900.8929443359375, + "KL/rejected_KL_mean": -1067.205322265625, + "KL/std": 515.5413818359375, + "epoch": 0.9486049926578561, + "fcm_dpo/beta": 0.0009276814525946975, + "fcm_dpo/delta": 0.09289085865020752, + "fcm_dpo/margin": 332.624755859375, + "fcm_dpo/q_t": 0.43122583627700806, + "grad_norm": 54.124847412109375, + "learning_rate": 4.256725079024553e-09, + "logits/chosen": -1.0459076166152954, + "logits/rejected": -1.022136926651001, + "logps/chosen": -795.8563232421875, + "logps/ref_chosen": -61.275787353515625, + "logps/ref_rejected": -77.50580596923828, + "logps/rejected": -1144.711181640625, + "loss": 1.184, + "margin_dpo/margin_mean": 332.624755859375, + "margin_dpo/margin_std": 628.484130859375, + "step": 646 + }, + { + "KL/chosen_KL_mean": -620.7327880859375, + "KL/mean": -821.4715576171875, + "KL/rejected_KL_mean": -1022.2103271484375, + "KL/std": 562.453125, + "epoch": 0.9500734214390602, + "fcm_dpo/beta": 0.0009334119968116283, + "fcm_dpo/delta": 0.025739368051290512, + "fcm_dpo/margin": 401.47760009765625, + "fcm_dpo/q_t": 0.4135919511318207, + "grad_norm": 33.10821533203125, + "learning_rate": 4.024152566816791e-09, + "logits/chosen": -0.9425860047340393, + "logits/rejected": -0.9679901599884033, + "logps/chosen": -675.585205078125, + "logps/ref_chosen": -54.8524169921875, + "logps/ref_rejected": -93.5194091796875, + "logps/rejected": -1115.729736328125, + "loss": 1.1112, + "margin_dpo/margin_mean": 401.47760009765625, + "margin_dpo/margin_std": 560.4625244140625, + "step": 647 + }, + { + "KL/chosen_KL_mean": -693.855224609375, + "KL/mean": -988.9381103515625, + "KL/rejected_KL_mean": -1284.02099609375, + "KL/std": 669.7720947265625, + "epoch": 0.9515418502202643, + "fcm_dpo/beta": 0.000923770247027278, + "fcm_dpo/delta": -0.14923778176307678, + "fcm_dpo/margin": 590.1658935546875, + "fcm_dpo/q_t": 0.3826466500759125, + "grad_norm": 28.995315551757812, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": -1.0780737400054932, + "logits/rejected": -1.1365818977355957, + "logps/chosen": -748.026611328125, + "logps/ref_chosen": -54.17146682739258, + "logps/ref_rejected": -98.7127914428711, + "logps/rejected": -1382.73388671875, + "loss": 1.024, + "margin_dpo/margin_mean": 590.1658935546875, + "margin_dpo/margin_std": 785.5125122070312, + "step": 648 + }, + { + "KL/chosen_KL_mean": -715.6250610351562, + "KL/mean": -850.495361328125, + "KL/rejected_KL_mean": -985.3656005859375, + "KL/std": 527.7951049804688, + "epoch": 0.9530102790014684, + "fcm_dpo/beta": 0.0009227098198607564, + "fcm_dpo/delta": 0.05385739728808403, + "fcm_dpo/margin": 269.74053955078125, + "fcm_dpo/q_t": 0.44609135389328003, + "grad_norm": 33.012882232666016, + "learning_rate": 3.5784585771215235e-09, + "logits/chosen": -1.1264615058898926, + "logits/rejected": -1.1146023273468018, + "logps/chosen": -778.1054077148438, + "logps/ref_chosen": -62.480350494384766, + "logps/ref_rejected": -80.07717895507812, + "logps/rejected": -1065.44287109375, + "loss": 1.2419, + "margin_dpo/margin_mean": 269.7405700683594, + "margin_dpo/margin_std": 652.0167236328125, + "step": 649 + }, + { + "KL/chosen_KL_mean": -752.7981567382812, + "KL/mean": -973.55517578125, + "KL/rejected_KL_mean": -1194.312255859375, + "KL/std": 645.759765625, + "epoch": 0.9544787077826725, + "fcm_dpo/beta": 0.0009236993500962853, + "fcm_dpo/delta": -0.008039167150855064, + "fcm_dpo/margin": 441.51409912109375, + "fcm_dpo/q_t": 0.4098489284515381, + "grad_norm": 28.0744571685791, + "learning_rate": 3.3653488440851253e-09, + "logits/chosen": -1.0201561450958252, + "logits/rejected": -1.0400092601776123, + "logps/chosen": -808.8909912109375, + "logps/ref_chosen": -56.09281921386719, + "logps/ref_rejected": -98.26483917236328, + "logps/rejected": -1292.5771484375, + "loss": 1.1285, + "margin_dpo/margin_mean": 441.51409912109375, + "margin_dpo/margin_std": 749.44873046875, + "step": 650 + }, + { + "KL/chosen_KL_mean": -520.7056884765625, + "KL/mean": -799.834228515625, + "KL/rejected_KL_mean": -1078.9627685546875, + "KL/std": 597.411865234375, + "epoch": 0.9559471365638766, + "fcm_dpo/beta": 0.0009163864306174219, + "fcm_dpo/delta": -0.11453430354595184, + "fcm_dpo/margin": 558.2569580078125, + "fcm_dpo/q_t": 0.3837953507900238, + "grad_norm": 45.06791687011719, + "learning_rate": 3.158738163478475e-09, + "logits/chosen": -1.0805871486663818, + "logits/rejected": -1.1315158605575562, + "logps/chosen": -564.1311645507812, + "logps/ref_chosen": -43.42544937133789, + "logps/ref_rejected": -99.95791625976562, + "logps/rejected": -1178.920654296875, + "loss": 1.0074, + "margin_dpo/margin_mean": 558.2569580078125, + "margin_dpo/margin_std": 605.560791015625, + "step": 651 + }, + { + "KL/chosen_KL_mean": -649.9320678710938, + "KL/mean": -872.9180908203125, + "KL/rejected_KL_mean": -1095.904052734375, + "KL/std": 631.821044921875, + "epoch": 0.9574155653450808, + "fcm_dpo/beta": 0.000913501949980855, + "fcm_dpo/delta": -0.007549758069217205, + "fcm_dpo/margin": 445.9720153808594, + "fcm_dpo/q_t": 0.4097236096858978, + "grad_norm": 41.35665512084961, + "learning_rate": 2.9586319796851555e-09, + "logits/chosen": -1.05128812789917, + "logits/rejected": -1.0744919776916504, + "logps/chosen": -712.5089111328125, + "logps/ref_chosen": -62.57680892944336, + "logps/ref_rejected": -111.76779174804688, + "logps/rejected": -1207.671875, + "loss": 1.1206, + "margin_dpo/margin_mean": 445.9720153808594, + "margin_dpo/margin_std": 735.88134765625, + "step": 652 + }, + { + "KL/chosen_KL_mean": -779.2682495117188, + "KL/mean": -977.2479858398438, + "KL/rejected_KL_mean": -1175.2276611328125, + "KL/std": 638.5052490234375, + "epoch": 0.9588839941262849, + "fcm_dpo/beta": 0.0009161117486655712, + "fcm_dpo/delta": 0.0379294790327549, + "fcm_dpo/margin": 395.95941162109375, + "fcm_dpo/q_t": 0.41951340436935425, + "grad_norm": 39.457916259765625, + "learning_rate": 2.7650355656892166e-09, + "logits/chosen": -1.0988178253173828, + "logits/rejected": -1.118626594543457, + "logps/chosen": -840.3812255859375, + "logps/ref_chosen": -61.11295700073242, + "logps/ref_rejected": -103.24960327148438, + "logps/rejected": -1278.477294921875, + "loss": 1.1425, + "margin_dpo/margin_mean": 395.95941162109375, + "margin_dpo/margin_std": 670.7147827148438, + "step": 653 + }, + { + "KL/chosen_KL_mean": -704.796875, + "KL/mean": -902.2587890625, + "KL/rejected_KL_mean": -1099.720703125, + "KL/std": 529.0486450195312, + "epoch": 0.960352422907489, + "fcm_dpo/beta": 0.000919223006349057, + "fcm_dpo/delta": 0.03766302019357681, + "fcm_dpo/margin": 394.923828125, + "fcm_dpo/q_t": 0.41876959800720215, + "grad_norm": 32.23603820800781, + "learning_rate": 2.577954022936174e-09, + "logits/chosen": -1.0824410915374756, + "logits/rejected": -1.1002042293548584, + "logps/chosen": -766.5250244140625, + "logps/ref_chosen": -61.7281379699707, + "logps/ref_rejected": -98.7738037109375, + "logps/rejected": -1198.4945068359375, + "loss": 1.1299, + "margin_dpo/margin_mean": 394.923828125, + "margin_dpo/margin_std": 613.5523681640625, + "step": 654 + }, + { + "KL/chosen_KL_mean": -666.8828125, + "KL/mean": -870.272705078125, + "KL/rejected_KL_mean": -1073.66259765625, + "KL/std": 549.2578735351562, + "epoch": 0.9618208516886931, + "fcm_dpo/beta": 0.0009220571955665946, + "fcm_dpo/delta": 0.025397051125764847, + "fcm_dpo/margin": 406.7799072265625, + "fcm_dpo/q_t": 0.41584494709968567, + "grad_norm": 34.27344512939453, + "learning_rate": 2.397392281198729e-09, + "logits/chosen": -1.0440378189086914, + "logits/rejected": -1.0873972177505493, + "logps/chosen": -716.4595947265625, + "logps/ref_chosen": -49.576812744140625, + "logps/ref_rejected": -98.29183197021484, + "logps/rejected": -1171.9544677734375, + "loss": 1.1277, + "margin_dpo/margin_mean": 406.7799072265625, + "margin_dpo/margin_std": 646.5740966796875, + "step": 655 + }, + { + "KL/chosen_KL_mean": -702.065673828125, + "KL/mean": -1036.923828125, + "KL/rejected_KL_mean": -1371.781982421875, + "KL/std": 697.924072265625, + "epoch": 0.9632892804698973, + "fcm_dpo/beta": 0.0009094408596865833, + "fcm_dpo/delta": -0.2153792828321457, + "fcm_dpo/margin": 669.7164306640625, + "fcm_dpo/q_t": 0.3670673072338104, + "grad_norm": 81.25033569335938, + "learning_rate": 2.223355098446622e-09, + "logits/chosen": -0.9668943881988525, + "logits/rejected": -1.0353293418884277, + "logps/chosen": -754.6151123046875, + "logps/ref_chosen": -52.54943084716797, + "logps/ref_rejected": -113.67464447021484, + "logps/rejected": -1485.456787109375, + "loss": 0.9595, + "margin_dpo/margin_mean": 669.7164306640625, + "margin_dpo/margin_std": 723.3271484375, + "step": 656 + }, + { + "KL/chosen_KL_mean": -684.0799560546875, + "KL/mean": -962.5208129882812, + "KL/rejected_KL_mean": -1240.961669921875, + "KL/std": 694.053466796875, + "epoch": 0.9647577092511013, + "fcm_dpo/beta": 0.0008950576884672046, + "fcm_dpo/delta": -0.10112009942531586, + "fcm_dpo/margin": 556.8817138671875, + "fcm_dpo/q_t": 0.38936832547187805, + "grad_norm": 43.923797607421875, + "learning_rate": 2.055847060721566e-09, + "logits/chosen": -1.0988727807998657, + "logits/rejected": -1.1430981159210205, + "logps/chosen": -730.780517578125, + "logps/ref_chosen": -46.700538635253906, + "logps/ref_rejected": -97.91487121582031, + "logps/rejected": -1338.87646484375, + "loss": 1.0394, + "margin_dpo/margin_mean": 556.8817138671875, + "margin_dpo/margin_std": 717.0012817382812, + "step": 657 + }, + { + "KL/chosen_KL_mean": -721.3847045898438, + "KL/mean": -944.3277587890625, + "KL/rejected_KL_mean": -1167.27099609375, + "KL/std": 536.6946411132812, + "epoch": 0.9662261380323054, + "fcm_dpo/beta": 0.0008913222118280828, + "fcm_dpo/delta": 0.002288389950990677, + "fcm_dpo/margin": 445.8861389160156, + "fcm_dpo/q_t": 0.4090738594532013, + "grad_norm": 33.75960159301758, + "learning_rate": 1.8948725820160662e-09, + "logits/chosen": -1.0495469570159912, + "logits/rejected": -1.0703763961791992, + "logps/chosen": -782.3428955078125, + "logps/ref_chosen": -60.95820999145508, + "logps/ref_rejected": -95.93949127197266, + "logps/rejected": -1263.21044921875, + "loss": 1.1046, + "margin_dpo/margin_mean": 445.8861083984375, + "margin_dpo/margin_std": 629.4903564453125, + "step": 658 + }, + { + "KL/chosen_KL_mean": -631.7986450195312, + "KL/mean": -848.4490966796875, + "KL/rejected_KL_mean": -1065.099609375, + "KL/std": 545.1578369140625, + "epoch": 0.9676945668135095, + "fcm_dpo/beta": 0.0008934010984376073, + "fcm_dpo/delta": 0.013058962300419807, + "fcm_dpo/margin": 433.3009033203125, + "fcm_dpo/q_t": 0.41191983222961426, + "grad_norm": 32.50177001953125, + "learning_rate": 1.7404359041573723e-09, + "logits/chosen": -0.9998725056648254, + "logits/rejected": -0.9671860933303833, + "logps/chosen": -708.5416259765625, + "logps/ref_chosen": -76.74298095703125, + "logps/ref_rejected": -87.4709701538086, + "logps/rejected": -1152.570556640625, + "loss": 1.1012, + "margin_dpo/margin_mean": 433.3009338378906, + "margin_dpo/margin_std": 592.5736083984375, + "step": 659 + }, + { + "KL/chosen_KL_mean": -696.3098754882812, + "KL/mean": -962.7720947265625, + "KL/rejected_KL_mean": -1229.234375, + "KL/std": 621.4789428710938, + "epoch": 0.9691629955947136, + "fcm_dpo/beta": 0.0008908716263249516, + "fcm_dpo/delta": -0.07654912024736404, + "fcm_dpo/margin": 532.9244995117188, + "fcm_dpo/q_t": 0.3918866515159607, + "grad_norm": 45.116607666015625, + "learning_rate": 1.592541096695571e-09, + "logits/chosen": -1.0789703130722046, + "logits/rejected": -1.089571237564087, + "logps/chosen": -755.3577880859375, + "logps/ref_chosen": -59.04788589477539, + "logps/ref_rejected": -75.96005249023438, + "logps/rejected": -1305.1944580078125, + "loss": 1.0467, + "margin_dpo/margin_mean": 532.9244995117188, + "margin_dpo/margin_std": 662.8816528320312, + "step": 660 + }, + { + "KL/chosen_KL_mean": -641.9539794921875, + "KL/mean": -880.4549560546875, + "KL/rejected_KL_mean": -1118.9559326171875, + "KL/std": 687.37255859375, + "epoch": 0.9706314243759178, + "fcm_dpo/beta": 0.000888321315869689, + "fcm_dpo/delta": -0.02433537133038044, + "fcm_dpo/margin": 477.001953125, + "fcm_dpo/q_t": 0.40677112340927124, + "grad_norm": 43.87881851196289, + "learning_rate": 1.4511920567963908e-09, + "logits/chosen": -1.1158676147460938, + "logits/rejected": -1.1237622499465942, + "logps/chosen": -692.6279296875, + "logps/ref_chosen": -50.673973083496094, + "logps/ref_rejected": -86.00569152832031, + "logps/rejected": -1204.961669921875, + "loss": 1.0893, + "margin_dpo/margin_mean": 477.001953125, + "margin_dpo/margin_std": 703.0227661132812, + "step": 661 + }, + { + "KL/chosen_KL_mean": -701.1622314453125, + "KL/mean": -907.7703857421875, + "KL/rejected_KL_mean": -1114.378662109375, + "KL/std": 595.3218994140625, + "epoch": 0.9720998531571219, + "fcm_dpo/beta": 0.0008879530942067504, + "fcm_dpo/delta": 0.03370830416679382, + "fcm_dpo/margin": 413.2164306640625, + "fcm_dpo/q_t": 0.4180566072463989, + "grad_norm": 28.001462936401367, + "learning_rate": 1.3163925091384532e-09, + "logits/chosen": -1.0258715152740479, + "logits/rejected": -1.023505449295044, + "logps/chosen": -770.4232788085938, + "logps/ref_chosen": -69.26106262207031, + "logps/ref_rejected": -89.05593872070312, + "logps/rejected": -1203.4345703125, + "loss": 1.1412, + "margin_dpo/margin_mean": 413.2164306640625, + "margin_dpo/margin_std": 703.9146728515625, + "step": 662 + }, + { + "KL/chosen_KL_mean": -683.2754516601562, + "KL/mean": -903.65966796875, + "KL/rejected_KL_mean": -1124.0439453125, + "KL/std": 640.8971557617188, + "epoch": 0.973568281938326, + "fcm_dpo/beta": 0.0008887812728062272, + "fcm_dpo/delta": 0.008383674547076225, + "fcm_dpo/margin": 440.7684020996094, + "fcm_dpo/q_t": 0.41332346200942993, + "grad_norm": 30.94547462463379, + "learning_rate": 1.1881460058152382e-09, + "logits/chosen": -1.0513726472854614, + "logits/rejected": -1.0720903873443604, + "logps/chosen": -748.1544189453125, + "logps/ref_chosen": -64.87890625, + "logps/ref_rejected": -113.92536926269531, + "logps/rejected": -1237.96923828125, + "loss": 1.1309, + "margin_dpo/margin_mean": 440.7684020996094, + "margin_dpo/margin_std": 747.7581787109375, + "step": 663 + }, + { + "KL/chosen_KL_mean": -748.7786254882812, + "KL/mean": -1007.6143798828125, + "KL/rejected_KL_mean": -1266.4501953125, + "KL/std": 679.8283081054688, + "epoch": 0.9750367107195301, + "fcm_dpo/beta": 0.0008842787938192487, + "fcm_dpo/delta": -0.05937542766332626, + "fcm_dpo/margin": 517.6715698242188, + "fcm_dpo/q_t": 0.39915400743484497, + "grad_norm": 30.055925369262695, + "learning_rate": 1.066455926241383e-09, + "logits/chosen": -1.0716259479522705, + "logits/rejected": -1.1003488302230835, + "logps/chosen": -809.6671142578125, + "logps/ref_chosen": -60.88847351074219, + "logps/ref_rejected": -105.521728515625, + "logps/rejected": -1371.971923828125, + "loss": 1.0709, + "margin_dpo/margin_mean": 517.6715698242188, + "margin_dpo/margin_std": 723.2990112304688, + "step": 664 + }, + { + "KL/chosen_KL_mean": -657.037109375, + "KL/mean": -874.8280029296875, + "KL/rejected_KL_mean": -1092.618896484375, + "KL/std": 550.8089599609375, + "epoch": 0.9765051395007343, + "fcm_dpo/beta": 0.0008855801424942911, + "fcm_dpo/delta": 0.014537844806909561, + "fcm_dpo/margin": 435.581787109375, + "fcm_dpo/q_t": 0.41070112586021423, + "grad_norm": 56.73260498046875, + "learning_rate": 9.513254770636137e-10, + "logits/chosen": -1.1636441946029663, + "logits/rejected": -1.1826890707015991, + "logps/chosen": -717.6011962890625, + "logps/ref_chosen": -60.56413269042969, + "logps/ref_rejected": -84.80882263183594, + "logps/rejected": -1177.427734375, + "loss": 1.0941, + "margin_dpo/margin_mean": 435.581787109375, + "margin_dpo/margin_std": 550.626708984375, + "step": 665 + }, + { + "KL/chosen_KL_mean": -698.910400390625, + "KL/mean": -909.08203125, + "KL/rejected_KL_mean": -1119.2535400390625, + "KL/std": 539.8720092773438, + "epoch": 0.9779735682819384, + "fcm_dpo/beta": 0.0008871962781995535, + "fcm_dpo/delta": 0.027589332312345505, + "fcm_dpo/margin": 420.3431396484375, + "fcm_dpo/q_t": 0.415420264005661, + "grad_norm": 30.555706024169922, + "learning_rate": 8.427576920763956e-10, + "logits/chosen": -0.9608547687530518, + "logits/rejected": -0.9632211923599243, + "logps/chosen": -763.3303833007812, + "logps/ref_chosen": -64.41996002197266, + "logps/ref_rejected": -95.8916244506836, + "logps/rejected": -1215.145263671875, + "loss": 1.1199, + "margin_dpo/margin_mean": 420.3431396484375, + "margin_dpo/margin_std": 612.9646606445312, + "step": 666 + }, + { + "KL/chosen_KL_mean": -784.1095581054688, + "KL/mean": -1053.423828125, + "KL/rejected_KL_mean": -1322.738037109375, + "KL/std": 630.7745361328125, + "epoch": 0.9794419970631424, + "fcm_dpo/beta": 0.0008815985638648272, + "fcm_dpo/delta": -0.07696354389190674, + "fcm_dpo/margin": 538.6284790039062, + "fcm_dpo/q_t": 0.3935006856918335, + "grad_norm": 44.02839660644531, + "learning_rate": 7.407554321417764e-10, + "logits/chosen": -0.9979052543640137, + "logits/rejected": -0.9953716993331909, + "logps/chosen": -853.3865966796875, + "logps/ref_chosen": -69.27702331542969, + "logps/ref_rejected": -87.83549499511719, + "logps/rejected": -1410.573486328125, + "loss": 1.0497, + "margin_dpo/margin_mean": 538.6284790039062, + "margin_dpo/margin_std": 688.3106689453125, + "step": 667 + }, + { + "KL/chosen_KL_mean": -829.9423217773438, + "KL/mean": -1009.3814697265625, + "KL/rejected_KL_mean": -1188.820556640625, + "KL/std": 652.5074462890625, + "epoch": 0.9809104258443465, + "fcm_dpo/beta": 0.0008891418110579252, + "fcm_dpo/delta": 0.0817713588476181, + "fcm_dpo/margin": 358.8783264160156, + "fcm_dpo/q_t": 0.4309133291244507, + "grad_norm": 50.39697265625, + "learning_rate": 6.453213851142225e-10, + "logits/chosen": -1.052807092666626, + "logits/rejected": -1.0501337051391602, + "logps/chosen": -902.5463256835938, + "logps/ref_chosen": -72.60400390625, + "logps/ref_rejected": -103.73905944824219, + "logps/rejected": -1292.5596923828125, + "loss": 1.2006, + "margin_dpo/margin_mean": 358.8783264160156, + "margin_dpo/margin_std": 768.2645263671875, + "step": 668 + }, + { + "KL/chosen_KL_mean": -619.5028076171875, + "KL/mean": -885.7294311523438, + "KL/rejected_KL_mean": -1151.9560546875, + "KL/std": 607.2215576171875, + "epoch": 0.9823788546255506, + "fcm_dpo/beta": 0.0008850685553625226, + "fcm_dpo/delta": -0.07294955104589462, + "fcm_dpo/margin": 532.453369140625, + "fcm_dpo/q_t": 0.39315131306648254, + "grad_norm": 24.650182723999023, + "learning_rate": 5.564580657695939e-10, + "logits/chosen": -1.0345218181610107, + "logits/rejected": -1.041704773902893, + "logps/chosen": -665.6192016601562, + "logps/ref_chosen": -46.116416931152344, + "logps/ref_rejected": -77.92434692382812, + "logps/rejected": -1229.8804931640625, + "loss": 1.054, + "margin_dpo/margin_mean": 532.453369140625, + "margin_dpo/margin_std": 684.6708984375, + "step": 669 + }, + { + "KL/chosen_KL_mean": -620.4657592773438, + "KL/mean": -882.550048828125, + "KL/rejected_KL_mean": -1144.6343994140625, + "KL/std": 581.8966674804688, + "epoch": 0.9838472834067548, + "fcm_dpo/beta": 0.0008802937809377909, + "fcm_dpo/delta": -0.06295409053564072, + "fcm_dpo/margin": 524.1685791015625, + "fcm_dpo/q_t": 0.3964523673057556, + "grad_norm": 27.264894485473633, + "learning_rate": 4.741678157389739e-10, + "logits/chosen": -0.9793117642402649, + "logits/rejected": -0.9908572435379028, + "logps/chosen": -682.8115234375, + "logps/ref_chosen": -62.34575271606445, + "logps/ref_rejected": -96.9405517578125, + "logps/rejected": -1241.574951171875, + "loss": 1.0678, + "margin_dpo/margin_mean": 524.1685791015625, + "margin_dpo/margin_std": 703.9255981445312, + "step": 670 + }, + { + "KL/chosen_KL_mean": -780.59716796875, + "KL/mean": -994.5520629882812, + "KL/rejected_KL_mean": -1208.5069580078125, + "KL/std": 590.3243408203125, + "epoch": 0.9853157121879589, + "fcm_dpo/beta": 0.000880036735907197, + "fcm_dpo/delta": 0.02367694117128849, + "fcm_dpo/margin": 427.90972900390625, + "fcm_dpo/q_t": 0.413568913936615, + "grad_norm": 35.296512603759766, + "learning_rate": 3.9845280344705245e-10, + "logits/chosen": -1.0900222063064575, + "logits/rejected": -1.1154989004135132, + "logps/chosen": -828.5972900390625, + "logps/ref_chosen": -48.00010681152344, + "logps/ref_rejected": -83.81932067871094, + "logps/rejected": -1292.3262939453125, + "loss": 1.1384, + "margin_dpo/margin_mean": 427.90972900390625, + "margin_dpo/margin_std": 720.406005859375, + "step": 671 + }, + { + "KL/chosen_KL_mean": -845.4471435546875, + "KL/mean": -1062.668701171875, + "KL/rejected_KL_mean": -1279.8902587890625, + "KL/std": 706.8720703125, + "epoch": 0.986784140969163, + "fcm_dpo/beta": 0.0008793792221695185, + "fcm_dpo/delta": 0.01829908788204193, + "fcm_dpo/margin": 434.44317626953125, + "fcm_dpo/q_t": 0.4161950945854187, + "grad_norm": 38.76650619506836, + "learning_rate": 3.293150240547549e-10, + "logits/chosen": -1.1482672691345215, + "logits/rejected": -1.1567609310150146, + "logps/chosen": -904.0303955078125, + "logps/ref_chosen": -58.58328628540039, + "logps/ref_rejected": -93.14015197753906, + "logps/rejected": -1373.030517578125, + "loss": 1.1456, + "margin_dpo/margin_mean": 434.4431457519531, + "margin_dpo/margin_std": 770.93408203125, + "step": 672 + }, + { + "KL/chosen_KL_mean": -725.902587890625, + "KL/mean": -928.3306274414062, + "KL/rejected_KL_mean": -1130.758544921875, + "KL/std": 572.4488525390625, + "epoch": 0.9882525697503671, + "fcm_dpo/beta": 0.0008829334983602166, + "fcm_dpo/delta": 0.04332459345459938, + "fcm_dpo/margin": 404.85595703125, + "fcm_dpo/q_t": 0.41951626539230347, + "grad_norm": 26.817668914794922, + "learning_rate": 2.6675629940689504e-10, + "logits/chosen": -1.091918706893921, + "logits/rejected": -1.0944968461990356, + "logps/chosen": -772.6258544921875, + "logps/ref_chosen": -46.72320556640625, + "logps/ref_rejected": -85.29623413085938, + "logps/rejected": -1216.0548095703125, + "loss": 1.1323, + "margin_dpo/margin_mean": 404.85595703125, + "margin_dpo/margin_std": 645.776611328125, + "step": 673 + }, + { + "KL/chosen_KL_mean": -647.2359008789062, + "KL/mean": -898.0692138671875, + "KL/rejected_KL_mean": -1148.902587890625, + "KL/std": 574.1219482421875, + "epoch": 0.9897209985315712, + "fcm_dpo/beta": 0.0008830685401335359, + "fcm_dpo/delta": -0.044042401015758514, + "fcm_dpo/margin": 501.666748046875, + "fcm_dpo/q_t": 0.4016549587249756, + "grad_norm": 40.91920852661133, + "learning_rate": 2.1077827798404725e-10, + "logits/chosen": -0.9943381547927856, + "logits/rejected": -1.0083208084106445, + "logps/chosen": -692.681396484375, + "logps/ref_chosen": -45.445526123046875, + "logps/ref_rejected": -70.04593658447266, + "logps/rejected": -1218.948486328125, + "loss": 1.075, + "margin_dpo/margin_mean": 501.666748046875, + "margin_dpo/margin_std": 687.1994018554688, + "step": 674 + }, + { + "KL/chosen_KL_mean": -708.86376953125, + "KL/mean": -968.4271240234375, + "KL/rejected_KL_mean": -1227.990478515625, + "KL/std": 640.6890258789062, + "epoch": 0.9911894273127754, + "fcm_dpo/beta": 0.0008735453011468053, + "fcm_dpo/delta": -0.05547531321644783, + "fcm_dpo/margin": 519.1265869140625, + "fcm_dpo/q_t": 0.3998996317386627, + "grad_norm": 34.21153259277344, + "learning_rate": 1.6138243485910863e-10, + "logits/chosen": -1.0351924896240234, + "logits/rejected": -1.0452386140823364, + "logps/chosen": -753.0400390625, + "logps/ref_chosen": -44.17628479003906, + "logps/ref_rejected": -74.09197998046875, + "logps/rejected": -1302.0823974609375, + "loss": 1.07, + "margin_dpo/margin_mean": 519.1265869140625, + "margin_dpo/margin_std": 684.087646484375, + "step": 675 + }, + { + "KL/chosen_KL_mean": -776.6356201171875, + "KL/mean": -1018.5604858398438, + "KL/rejected_KL_mean": -1260.4853515625, + "KL/std": 617.2457885742188, + "epoch": 0.9926578560939795, + "fcm_dpo/beta": 0.0008749394328333437, + "fcm_dpo/delta": -0.023866407573223114, + "fcm_dpo/margin": 483.8497619628906, + "fcm_dpo/q_t": 0.40422454476356506, + "grad_norm": 28.231639862060547, + "learning_rate": 1.1857007165852472e-10, + "logits/chosen": -1.0005314350128174, + "logits/rejected": -1.0063731670379639, + "logps/chosen": -848.0341186523438, + "logps/ref_chosen": -71.39852905273438, + "logps/ref_rejected": -88.3587646484375, + "logps/rejected": -1348.8441162109375, + "loss": 1.0782, + "margin_dpo/margin_mean": 483.8497314453125, + "margin_dpo/margin_std": 629.48583984375, + "step": 676 + }, + { + "KL/chosen_KL_mean": -752.8453979492188, + "KL/mean": -1003.5128173828125, + "KL/rejected_KL_mean": -1254.1802978515625, + "KL/std": 636.5200805664062, + "epoch": 0.9941262848751835, + "fcm_dpo/beta": 0.000870502321049571, + "fcm_dpo/delta": -0.037262238562107086, + "fcm_dpo/margin": 501.3348693847656, + "fcm_dpo/q_t": 0.4067116677761078, + "grad_norm": 33.513336181640625, + "learning_rate": 8.23423165278725e-11, + "logits/chosen": -1.0794928073883057, + "logits/rejected": -1.0712941884994507, + "logps/chosen": -809.372802734375, + "logps/ref_chosen": -56.527435302734375, + "logps/ref_rejected": -78.22654724121094, + "logps/rejected": -1332.40673828125, + "loss": 1.0906, + "margin_dpo/margin_mean": 501.3348693847656, + "margin_dpo/margin_std": 767.4678344726562, + "step": 677 + }, + { + "KL/chosen_KL_mean": -626.833251953125, + "KL/mean": -919.983642578125, + "KL/rejected_KL_mean": -1213.134033203125, + "KL/std": 699.7562255859375, + "epoch": 0.9955947136563876, + "fcm_dpo/beta": 0.0008654722478240728, + "fcm_dpo/delta": -0.1102285087108612, + "fcm_dpo/margin": 586.30078125, + "fcm_dpo/q_t": 0.3892865777015686, + "grad_norm": 35.17583465576172, + "learning_rate": 5.270012410216185e-11, + "logits/chosen": -1.0084434747695923, + "logits/rejected": -1.0442242622375488, + "logps/chosen": -672.9677734375, + "logps/ref_chosen": -46.13447570800781, + "logps/ref_rejected": -80.60462951660156, + "logps/rejected": -1293.73876953125, + "loss": 1.0515, + "margin_dpo/margin_mean": 586.30078125, + "margin_dpo/margin_std": 820.827392578125, + "step": 678 + }, + { + "KL/chosen_KL_mean": -740.3648681640625, + "KL/mean": -935.5812377929688, + "KL/rejected_KL_mean": -1130.797607421875, + "KL/std": 556.906005859375, + "epoch": 0.9970631424375918, + "fcm_dpo/beta": 0.000864343426655978, + "fcm_dpo/delta": 0.06362688541412354, + "fcm_dpo/margin": 390.4327087402344, + "fcm_dpo/q_t": 0.4237492084503174, + "grad_norm": 33.905616760253906, + "learning_rate": 2.9644275480772416e-11, + "logits/chosen": -1.0304598808288574, + "logits/rejected": -1.0171773433685303, + "logps/chosen": -790.6597900390625, + "logps/ref_chosen": -50.294921875, + "logps/ref_rejected": -76.59813690185547, + "logps/rejected": -1207.395751953125, + "loss": 1.1458, + "margin_dpo/margin_mean": 390.4327087402344, + "margin_dpo/margin_std": 631.3106079101562, + "step": 679 + }, + { + "KL/chosen_KL_mean": -745.6943359375, + "KL/mean": -1011.821533203125, + "KL/rejected_KL_mean": -1277.94873046875, + "KL/std": 722.32958984375, + "epoch": 0.9985315712187959, + "fcm_dpo/beta": 0.0008604623144492507, + "fcm_dpo/delta": -0.059656865894794464, + "fcm_dpo/margin": 532.2544555664062, + "fcm_dpo/q_t": 0.39625048637390137, + "grad_norm": 34.24609375, + "learning_rate": 1.31753782067201e-11, + "logits/chosen": -1.0160003900527954, + "logits/rejected": -1.0428314208984375, + "logps/chosen": -822.6099853515625, + "logps/ref_chosen": -76.91569519042969, + "logps/ref_rejected": -112.384765625, + "logps/rejected": -1390.33349609375, + "loss": 1.0828, + "margin_dpo/margin_mean": 532.2544555664062, + "margin_dpo/margin_std": 783.269287109375, + "step": 680 + }, + { + "KL/chosen_KL_mean": -750.117919921875, + "KL/mean": -952.0081787109375, + "KL/rejected_KL_mean": -1153.8985595703125, + "KL/std": 590.4039306640625, + "epoch": 1.0, + "fcm_dpo/beta": 0.0008671648683957756, + "fcm_dpo/delta": 0.05011419206857681, + "fcm_dpo/margin": 403.78057861328125, + "fcm_dpo/q_t": 0.4205058813095093, + "grad_norm": 33.01895523071289, + "learning_rate": 3.2938662507808745e-12, + "logits/chosen": -1.0952690839767456, + "logits/rejected": -1.1073505878448486, + "logps/chosen": -811.0751953125, + "logps/ref_chosen": -60.957279205322266, + "logps/ref_rejected": -88.55797576904297, + "logps/rejected": -1242.45654296875, + "loss": 1.1452, + "margin_dpo/margin_mean": 403.7805480957031, + "margin_dpo/margin_std": 639.1034545898438, + "step": 681 + }, + { + "epoch": 1.0, + "step": 681, + "total_flos": 0.0, + "train_loss": 1.0870012780118095, + "train_runtime": 1735.4876, + "train_samples_per_second": 25.121, + "train_steps_per_second": 0.392 + } + ], + "logging_steps": 1, + "max_steps": 681, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}