commit af7d6761b464880a16778b186aef126b2453a2b2 Author: ModelHub XC Date: Fri May 22 18:39:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: W-61/llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a40c14 --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +--- +library_name: transformers +base_model: W-61/llama-3-8b-base-sft-hh-harmless-4xh200 +tags: +- alignment-handbook +- new-dpo +- generated_from_trainer +datasets: +- Anthropic/hh-rlhf +model-index: +- name: llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 + results: [] +--- + + + +# llama-3-8b-base-new-dpo-hh-harmless-4xh200-batch-64-q_t-0.45-s_star-0.4-eta-0.01 + +This model is a fine-tuned version of [W-61/llama-3-8b-base-sft-hh-harmless-4xh200](https://huggingface.co/W-61/llama-3-8b-base-sft-hh-harmless-4xh200) on the Anthropic/hh-rlhf dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-07 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 64 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 1 + +### Training results + + + +### Framework versions + +- Transformers 4.51.0 +- Pytorch 2.3.1+cu121 +- Datasets 2.21.0 +- Tokenizers 0.21.4 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..7ee8e28 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1039659273606386, + "train_runtime": 1648.6122, + "train_samples": 42336, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..5092b09 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.51.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..76247c9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "max_length": 4096, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.0" +} diff --git a/model-00001-of-00007.safetensors b/model-00001-of-00007.safetensors new file mode 100644 index 0000000..4f9f434 --- /dev/null +++ b/model-00001-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd7a4ec1316f3f2b3841569063fe7288d994744aaa86468b327394cdbafca98 +size 4886466168 diff --git a/model-00002-of-00007.safetensors b/model-00002-of-00007.safetensors new file mode 100644 index 0000000..cb1fa33 --- /dev/null +++ b/model-00002-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cf78d867bf7fcea51e8a5c025417653a215c70fe530f6598d37a2d245040198 +size 4832007448 diff --git a/model-00003-of-00007.safetensors b/model-00003-of-00007.safetensors new file mode 100644 index 0000000..710bf20 --- /dev/null +++ b/model-00003-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51ba0834894eeb1d22fca6cd00d7afe65b3ae8d97c8ae16df225add2106f2b7 +size 4999813112 diff --git a/model-00004-of-00007.safetensors b/model-00004-of-00007.safetensors new file mode 100644 index 0000000..e997836 --- /dev/null +++ b/model-00004-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7738c56cfb0beec8bcc934740f36569dcd489ffd0137db28ecb82128fa15e0b +size 4999813128 diff --git a/model-00005-of-00007.safetensors b/model-00005-of-00007.safetensors new file mode 100644 index 0000000..9c51347 --- /dev/null +++ b/model-00005-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:089fa52e3232c3526d45ddcc141a87e0e4a911d727750efc54ec7e18065f4097 +size 4832007496 diff --git a/model-00006-of-00007.safetensors b/model-00006-of-00007.safetensors new file mode 100644 index 0000000..3823477 --- /dev/null +++ b/model-00006-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0888524c3cb11a736f1bba3c5908f6b4d436a0bbdbf248393ddf451946cc898e +size 4999813120 diff --git a/model-00007-of-00007.safetensors b/model-00007-of-00007.safetensors new file mode 100644 index 0000000..75d1087 --- /dev/null +++ b/model-00007-of-00007.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b98a6cd6660ea54702610661f5aad8fa85fd6f7902781141484bd54513b59c1 +size 2571158184 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0985084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 32121044992 + }, + "weight_map": { + "lm_head.weight": "model-00007-of-00007.safetensors", + "model.embed_tokens.weight": "model-00001-of-00007.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors", + "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors", + "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors", + "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors", + "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors", + "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors", + "model.norm.weight": "model-00007-of-00007.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..86a3394 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393 +size 17209961 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8c6916a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 2048, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..7ee8e28 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.999244142101285, + "total_flos": 0.0, + "train_loss": 1.1039659273606386, + "train_runtime": 1648.6122, + "train_samples": 42336, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..31e4d6a --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,15246 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.999244142101285, + "eval_steps": 200, + "global_step": 661, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "KL/chosen_KL_mean": 0.02867889404296875, + "KL/mean": 0.029354453086853027, + "KL/rejected_KL_mean": 0.030029296875, + "KL/std": 0.2071000635623932, + "epoch": 0.0015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0013532638549804688, + "fcm_dpo/q_t": 0.5000336766242981, + "grad_norm": 28.219579696655273, + "learning_rate": 0.0, + "logits/chosen": 0.13337239623069763, + "logits/rejected": 0.12492949515581131, + "logps/chosen": -64.5841293334961, + "logps/ref_chosen": -64.61280822753906, + "logps/ref_rejected": -64.17195129394531, + "logps/rejected": -64.14192199707031, + "loss": 1.3866, + "margin_dpo/margin_mean": -0.0013527870178222656, + "margin_dpo/margin_std": 0.2561596930027008, + "step": 1 + }, + { + "KL/chosen_KL_mean": -0.00289154052734375, + "KL/mean": -0.021616414189338684, + "KL/rejected_KL_mean": -0.04033660888671875, + "KL/std": 0.19624735414981842, + "epoch": 0.0030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.037450045347213745, + "fcm_dpo/q_t": 0.4990639090538025, + "grad_norm": 27.82788848876953, + "learning_rate": 7.462686567164179e-09, + "logits/chosen": 0.09414851665496826, + "logits/rejected": 0.07363267242908478, + "logps/chosen": -56.101890563964844, + "logps/ref_chosen": -56.0989990234375, + "logps/ref_rejected": -66.59971618652344, + "logps/rejected": -66.64006042480469, + "loss": 1.3828, + "margin_dpo/margin_mean": 0.03744968771934509, + "margin_dpo/margin_std": 0.27811938524246216, + "step": 2 + }, + { + "KL/chosen_KL_mean": 0.01413726806640625, + "KL/mean": 0.025065720081329346, + "KL/rejected_KL_mean": 0.035991668701171875, + "KL/std": 0.28161460161209106, + "epoch": 0.0045351473922902496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.021856069564819336, + "fcm_dpo/q_t": 0.5005451440811157, + "grad_norm": 31.18069076538086, + "learning_rate": 1.4925373134328357e-08, + "logits/chosen": 0.09933006763458252, + "logits/rejected": 0.06128678470849991, + "logps/chosen": -65.44312286376953, + "logps/ref_chosen": -65.45726013183594, + "logps/ref_rejected": -90.82853698730469, + "logps/rejected": -90.79254150390625, + "loss": 1.3888, + "margin_dpo/margin_mean": -0.021855950355529785, + "margin_dpo/margin_std": 0.3706222176551819, + "step": 3 + }, + { + "KL/chosen_KL_mean": 0.049053192138671875, + "KL/mean": 0.04321002960205078, + "KL/rejected_KL_mean": 0.03736114501953125, + "KL/std": 0.22751623392105103, + "epoch": 0.006046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.011696159839630127, + "fcm_dpo/q_t": 0.4997084140777588, + "grad_norm": 34.37651062011719, + "learning_rate": 2.2388059701492534e-08, + "logits/chosen": 0.1125967800617218, + "logits/rejected": 0.09639419615268707, + "logps/chosen": -76.81112670898438, + "logps/ref_chosen": -76.86018371582031, + "logps/ref_rejected": -79.91523742675781, + "logps/rejected": -79.87787628173828, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.011695504188537598, + "margin_dpo/margin_std": 0.331972599029541, + "step": 4 + }, + { + "KL/chosen_KL_mean": -0.016885757446289062, + "KL/mean": 0.00026844441890716553, + "KL/rejected_KL_mean": 0.017429351806640625, + "KL/std": 0.24213802814483643, + "epoch": 0.007558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0343116819858551, + "fcm_dpo/q_t": 0.5008580684661865, + "grad_norm": 29.672016143798828, + "learning_rate": 2.9850746268656714e-08, + "logits/chosen": 0.08228074014186859, + "logits/rejected": 0.043413855135440826, + "logps/chosen": -62.98822784423828, + "logps/ref_chosen": -62.97134017944336, + "logps/ref_rejected": -79.9192123413086, + "logps/rejected": -79.90177917480469, + "loss": 1.3901, + "margin_dpo/margin_mean": -0.03431084752082825, + "margin_dpo/margin_std": 0.35833635926246643, + "step": 5 + }, + { + "KL/chosen_KL_mean": -0.07754325866699219, + "KL/mean": -0.02470727264881134, + "KL/rejected_KL_mean": 0.02812957763671875, + "KL/std": 0.23891785740852356, + "epoch": 0.009070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.10566946864128113, + "fcm_dpo/q_t": 0.5026400089263916, + "grad_norm": 30.006505966186523, + "learning_rate": 3.731343283582089e-08, + "logits/chosen": 0.14933988451957703, + "logits/rejected": 0.10950936377048492, + "logps/chosen": -51.38490676879883, + "logps/ref_chosen": -51.30736541748047, + "logps/ref_rejected": -82.77239227294922, + "logps/rejected": -82.7442626953125, + "loss": 1.3972, + "margin_dpo/margin_mean": -0.10566920042037964, + "margin_dpo/margin_std": 0.33293482661247253, + "step": 6 + }, + { + "KL/chosen_KL_mean": 0.01363372802734375, + "KL/mean": 0.003945142030715942, + "KL/rejected_KL_mean": -0.00574493408203125, + "KL/std": 0.21995863318443298, + "epoch": 0.010582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01937735080718994, + "fcm_dpo/q_t": 0.49951550364494324, + "grad_norm": 27.190397262573242, + "learning_rate": 4.477611940298507e-08, + "logits/chosen": 0.008923820219933987, + "logits/rejected": -0.034440357238054276, + "logps/chosen": -51.445777893066406, + "logps/ref_chosen": -51.45941162109375, + "logps/ref_rejected": -66.3828125, + "logps/rejected": -66.3885498046875, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.019377440214157104, + "margin_dpo/margin_std": 0.2998865246772766, + "step": 7 + }, + { + "KL/chosen_KL_mean": 0.011030197143554688, + "KL/mean": 0.003461584448814392, + "KL/rejected_KL_mean": -0.0041046142578125, + "KL/std": 0.2384001612663269, + "epoch": 0.012093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.015131622552871704, + "fcm_dpo/q_t": 0.4996224045753479, + "grad_norm": 28.706321716308594, + "learning_rate": 5.223880597014925e-08, + "logits/chosen": 0.07867727428674698, + "logits/rejected": 0.056360866874456406, + "logps/chosen": -62.18651580810547, + "logps/ref_chosen": -62.197547912597656, + "logps/ref_rejected": -74.66180419921875, + "logps/rejected": -74.66590881347656, + "loss": 1.3851, + "margin_dpo/margin_mean": 0.015131056308746338, + "margin_dpo/margin_std": 0.3448169231414795, + "step": 8 + }, + { + "KL/chosen_KL_mean": -0.029148101806640625, + "KL/mean": -0.04673531651496887, + "KL/rejected_KL_mean": -0.06432342529296875, + "KL/std": 0.2524610757827759, + "epoch": 0.013605442176870748, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03517633676528931, + "fcm_dpo/q_t": 0.4991207718849182, + "grad_norm": 31.5657901763916, + "learning_rate": 5.970149253731343e-08, + "logits/chosen": 0.14904074370861053, + "logits/rejected": 0.09135086834430695, + "logps/chosen": -55.65886688232422, + "logps/ref_chosen": -55.629722595214844, + "logps/ref_rejected": -86.21221923828125, + "logps/rejected": -86.27655029296875, + "loss": 1.3832, + "margin_dpo/margin_mean": 0.03517654538154602, + "margin_dpo/margin_std": 0.3983437716960907, + "step": 9 + }, + { + "KL/chosen_KL_mean": -0.02042388916015625, + "KL/mean": -0.009125128388404846, + "KL/rejected_KL_mean": 0.002170562744140625, + "KL/std": 0.2475941777229309, + "epoch": 0.015117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.022598236799240112, + "fcm_dpo/q_t": 0.5005645751953125, + "grad_norm": 29.73713493347168, + "learning_rate": 6.71641791044776e-08, + "logits/chosen": 0.12513160705566406, + "logits/rejected": 0.09449122101068497, + "logps/chosen": -62.71102523803711, + "logps/ref_chosen": -62.69060134887695, + "logps/ref_rejected": -90.610107421875, + "logps/rejected": -90.60794067382812, + "loss": 1.3889, + "margin_dpo/margin_mean": -0.022598326206207275, + "margin_dpo/margin_std": 0.3694617748260498, + "step": 10 + }, + { + "KL/chosen_KL_mean": -0.01373291015625, + "KL/mean": -0.027163296937942505, + "KL/rejected_KL_mean": -0.04059600830078125, + "KL/std": 0.17830416560173035, + "epoch": 0.016628873771730914, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.026863932609558105, + "fcm_dpo/q_t": 0.4993281960487366, + "grad_norm": 29.407014846801758, + "learning_rate": 7.462686567164178e-08, + "logits/chosen": 0.11649392545223236, + "logits/rejected": 0.10949207842350006, + "logps/chosen": -65.78085327148438, + "logps/ref_chosen": -65.76712036132812, + "logps/ref_rejected": -72.4764633178711, + "logps/rejected": -72.51705932617188, + "loss": 1.3838, + "margin_dpo/margin_mean": 0.02686440944671631, + "margin_dpo/margin_std": 0.2555994987487793, + "step": 11 + }, + { + "KL/chosen_KL_mean": -0.022777557373046875, + "KL/mean": -0.03335832059383392, + "KL/rejected_KL_mean": -0.04393959045410156, + "KL/std": 0.1987801492214203, + "epoch": 0.018140589569160998, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.02116510272026062, + "fcm_dpo/q_t": 0.49947118759155273, + "grad_norm": 28.210813522338867, + "learning_rate": 8.208955223880596e-08, + "logits/chosen": 0.029278969392180443, + "logits/rejected": 0.012985341250896454, + "logps/chosen": -60.72766876220703, + "logps/ref_chosen": -60.704891204833984, + "logps/ref_rejected": -69.41564178466797, + "logps/rejected": -69.45957946777344, + "loss": 1.3844, + "margin_dpo/margin_mean": 0.021165400743484497, + "margin_dpo/margin_std": 0.29928159713745117, + "step": 12 + }, + { + "KL/chosen_KL_mean": 0.011051177978515625, + "KL/mean": 0.039337918162345886, + "KL/rejected_KL_mean": 0.06763076782226562, + "KL/std": 0.205886572599411, + "epoch": 0.019652305366591082, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.056578487157821655, + "fcm_dpo/q_t": 0.5014140605926514, + "grad_norm": 29.18099594116211, + "learning_rate": 8.955223880597014e-08, + "logits/chosen": 0.11636652052402496, + "logits/rejected": 0.05318449065089226, + "logps/chosen": -49.8982048034668, + "logps/ref_chosen": -49.90925598144531, + "logps/ref_rejected": -92.37818145751953, + "logps/rejected": -92.310546875, + "loss": 1.3922, + "margin_dpo/margin_mean": -0.0565793514251709, + "margin_dpo/margin_std": 0.2904546856880188, + "step": 13 + }, + { + "KL/chosen_KL_mean": 0.018739700317382812, + "KL/mean": -0.012100264430046082, + "KL/rejected_KL_mean": -0.042934417724609375, + "KL/std": 0.2116500735282898, + "epoch": 0.021164021164021163, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06167241930961609, + "fcm_dpo/q_t": 0.49845898151397705, + "grad_norm": 29.35123062133789, + "learning_rate": 9.701492537313432e-08, + "logits/chosen": 0.07167838513851166, + "logits/rejected": 0.054354965686798096, + "logps/chosen": -60.60005187988281, + "logps/ref_chosen": -60.61879348754883, + "logps/ref_rejected": -71.79306030273438, + "logps/rejected": -71.83599853515625, + "loss": 1.3804, + "margin_dpo/margin_mean": 0.06167188286781311, + "margin_dpo/margin_std": 0.32041066884994507, + "step": 14 + }, + { + "KL/chosen_KL_mean": -0.016628265380859375, + "KL/mean": -0.030201956629753113, + "KL/rejected_KL_mean": -0.043773651123046875, + "KL/std": 0.23075683414936066, + "epoch": 0.022675736961451247, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.027143627405166626, + "fcm_dpo/q_t": 0.49932122230529785, + "grad_norm": 33.13405990600586, + "learning_rate": 1.044776119402985e-07, + "logits/chosen": 0.08718998730182648, + "logits/rejected": 0.04282126948237419, + "logps/chosen": -63.486167907714844, + "logps/ref_chosen": -63.46953582763672, + "logps/ref_rejected": -88.88951110839844, + "logps/rejected": -88.93328857421875, + "loss": 1.3838, + "margin_dpo/margin_mean": 0.027143627405166626, + "margin_dpo/margin_std": 0.3156331777572632, + "step": 15 + }, + { + "KL/chosen_KL_mean": 0.025264739990234375, + "KL/mean": 0.015766292810440063, + "KL/rejected_KL_mean": 0.00626373291015625, + "KL/std": 0.20275935530662537, + "epoch": 0.02418745275888133, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.01899963617324829, + "fcm_dpo/q_t": 0.4995252788066864, + "grad_norm": 26.81545066833496, + "learning_rate": 1.1194029850746268e-07, + "logits/chosen": 0.08911541104316711, + "logits/rejected": 0.05375181883573532, + "logps/chosen": -46.50703430175781, + "logps/ref_chosen": -46.53229904174805, + "logps/ref_rejected": -74.27533721923828, + "logps/rejected": -74.26907348632812, + "loss": 1.3846, + "margin_dpo/margin_mean": 0.01899990439414978, + "margin_dpo/margin_std": 0.2597685754299164, + "step": 16 + }, + { + "KL/chosen_KL_mean": -0.03475761413574219, + "KL/mean": -0.019420340657234192, + "KL/rejected_KL_mean": -0.00408172607421875, + "KL/std": 0.2204478681087494, + "epoch": 0.025699168556311415, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.03067687153816223, + "fcm_dpo/q_t": 0.5007668733596802, + "grad_norm": 33.07863235473633, + "learning_rate": 1.1940298507462686e-07, + "logits/chosen": 0.0719546377658844, + "logits/rejected": 0.05250099301338196, + "logps/chosen": -64.11259460449219, + "logps/ref_chosen": -64.07783508300781, + "logps/ref_rejected": -86.40876770019531, + "logps/rejected": -86.41284942626953, + "loss": 1.3896, + "margin_dpo/margin_mean": -0.03067764639854431, + "margin_dpo/margin_std": 0.2964497208595276, + "step": 17 + }, + { + "KL/chosen_KL_mean": 0.0003871917724609375, + "KL/mean": -0.0026166439056396484, + "KL/rejected_KL_mean": -0.00562286376953125, + "KL/std": 0.22112533450126648, + "epoch": 0.027210884353741496, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.006009459495544434, + "fcm_dpo/q_t": 0.4998498558998108, + "grad_norm": 27.938682556152344, + "learning_rate": 1.2686567164179106e-07, + "logits/chosen": 0.08263183385133743, + "logits/rejected": 0.03783968836069107, + "logps/chosen": -44.87394714355469, + "logps/ref_chosen": -44.87433624267578, + "logps/ref_rejected": -70.97604370117188, + "logps/rejected": -70.98167419433594, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.006009340286254883, + "margin_dpo/margin_std": 0.3140023946762085, + "step": 18 + }, + { + "KL/chosen_KL_mean": 0.024576187133789062, + "KL/mean": -0.0010314956307411194, + "KL/rejected_KL_mean": -0.026641845703125, + "KL/std": 0.22808700799942017, + "epoch": 0.02872260015117158, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05122239887714386, + "fcm_dpo/q_t": 0.4987197816371918, + "grad_norm": 30.819204330444336, + "learning_rate": 1.343283582089552e-07, + "logits/chosen": 0.054087694734334946, + "logits/rejected": 0.040823794901371, + "logps/chosen": -68.13523864746094, + "logps/ref_chosen": -68.1598129272461, + "logps/ref_rejected": -81.17138671875, + "logps/rejected": -81.19802856445312, + "loss": 1.3814, + "margin_dpo/margin_mean": 0.051222607493400574, + "margin_dpo/margin_std": 0.304071307182312, + "step": 19 + }, + { + "KL/chosen_KL_mean": 0.0057506561279296875, + "KL/mean": 0.0002925768494606018, + "KL/rejected_KL_mean": -0.005161285400390625, + "KL/std": 0.20305666327476501, + "epoch": 0.030234315948601664, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.010908856987953186, + "fcm_dpo/q_t": 0.4997273087501526, + "grad_norm": 29.17350196838379, + "learning_rate": 1.4179104477611938e-07, + "logits/chosen": 0.12288285791873932, + "logits/rejected": 0.10042296350002289, + "logps/chosen": -53.672813415527344, + "logps/ref_chosen": -53.67856216430664, + "logps/ref_rejected": -74.16911315917969, + "logps/rejected": -74.17427825927734, + "loss": 1.3854, + "margin_dpo/margin_mean": 0.01090879738330841, + "margin_dpo/margin_std": 0.2811530828475952, + "step": 20 + }, + { + "KL/chosen_KL_mean": 0.018339157104492188, + "KL/mean": 0.004051558673381805, + "KL/rejected_KL_mean": -0.010234832763671875, + "KL/std": 0.21852260828018188, + "epoch": 0.031746031746031744, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.028580710291862488, + "fcm_dpo/q_t": 0.49928581714630127, + "grad_norm": 29.2486629486084, + "learning_rate": 1.4925373134328355e-07, + "logits/chosen": 0.13036969304084778, + "logits/rejected": 0.10371002554893494, + "logps/chosen": -64.68321228027344, + "logps/ref_chosen": -64.70155334472656, + "logps/ref_rejected": -81.02095031738281, + "logps/rejected": -81.03118133544922, + "loss": 1.3836, + "margin_dpo/margin_mean": 0.02857983112335205, + "margin_dpo/margin_std": 0.2810309827327728, + "step": 21 + }, + { + "KL/chosen_KL_mean": 0.0059661865234375, + "KL/mean": -0.0203571617603302, + "KL/rejected_KL_mean": -0.046680450439453125, + "KL/std": 0.1862630546092987, + "epoch": 0.03325774754346183, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.052648723125457764, + "fcm_dpo/q_t": 0.4986839294433594, + "grad_norm": 28.90389633178711, + "learning_rate": 1.5671641791044775e-07, + "logits/chosen": -0.012446942739188671, + "logits/rejected": -0.03257812187075615, + "logps/chosen": -58.030025482177734, + "logps/ref_chosen": -58.03599166870117, + "logps/ref_rejected": -80.72721862792969, + "logps/rejected": -80.77389526367188, + "loss": 1.3812, + "margin_dpo/margin_mean": 0.052648723125457764, + "margin_dpo/margin_std": 0.25717687606811523, + "step": 22 + }, + { + "KL/chosen_KL_mean": -0.0080718994140625, + "KL/mean": 0.0033599436283111572, + "KL/rejected_KL_mean": 0.0147857666015625, + "KL/std": 0.23230639100074768, + "epoch": 0.03476946334089191, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.022857844829559326, + "fcm_dpo/q_t": 0.5005712509155273, + "grad_norm": 33.04319381713867, + "learning_rate": 1.6417910447761193e-07, + "logits/chosen": 0.15227936208248138, + "logits/rejected": 0.12623241543769836, + "logps/chosen": -66.3641586303711, + "logps/ref_chosen": -66.35608673095703, + "logps/ref_rejected": -93.02769470214844, + "logps/rejected": -93.01290893554688, + "loss": 1.3888, + "margin_dpo/margin_mean": -0.022858083248138428, + "margin_dpo/margin_std": 0.2870565950870514, + "step": 23 + }, + { + "KL/chosen_KL_mean": -0.05757331848144531, + "KL/mean": -0.02590470016002655, + "KL/rejected_KL_mean": 0.0057621002197265625, + "KL/std": 0.2218666672706604, + "epoch": 0.036281179138321996, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.06333401799201965, + "fcm_dpo/q_t": 0.5015828013420105, + "grad_norm": 26.447811126708984, + "learning_rate": 1.716417910447761e-07, + "logits/chosen": 0.12736168503761292, + "logits/rejected": 0.09465532004833221, + "logps/chosen": -54.51881408691406, + "logps/ref_chosen": -54.461238861083984, + "logps/ref_rejected": -68.33817291259766, + "logps/rejected": -68.33241271972656, + "loss": 1.3928, + "margin_dpo/margin_mean": -0.0633348822593689, + "margin_dpo/margin_std": 0.287397563457489, + "step": 24 + }, + { + "KL/chosen_KL_mean": -0.03167724609375, + "KL/mean": -0.04313118755817413, + "KL/rejected_KL_mean": -0.0545806884765625, + "KL/std": 0.2588927745819092, + "epoch": 0.03779289493575208, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.022903114557266235, + "fcm_dpo/q_t": 0.49942734837532043, + "grad_norm": 29.668231964111328, + "learning_rate": 1.7910447761194027e-07, + "logits/chosen": 0.11026899516582489, + "logits/rejected": 0.05842104181647301, + "logps/chosen": -60.03588104248047, + "logps/ref_chosen": -60.00420379638672, + "logps/ref_rejected": -90.47376251220703, + "logps/rejected": -90.5283432006836, + "loss": 1.3843, + "margin_dpo/margin_mean": 0.022902995347976685, + "margin_dpo/margin_std": 0.3221206068992615, + "step": 25 + }, + { + "KL/chosen_KL_mean": -0.0179595947265625, + "KL/mean": -0.012291766703128815, + "KL/rejected_KL_mean": -0.006626129150390625, + "KL/std": 0.20800243318080902, + "epoch": 0.039304610733182165, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.011333361268043518, + "fcm_dpo/q_t": 0.5002831816673279, + "grad_norm": 29.86821937561035, + "learning_rate": 1.8656716417910447e-07, + "logits/chosen": 0.1290886253118515, + "logits/rejected": 0.11009064316749573, + "logps/chosen": -56.83711242675781, + "logps/ref_chosen": -56.81915283203125, + "logps/ref_rejected": -77.84333038330078, + "logps/rejected": -77.84996032714844, + "loss": 1.3877, + "margin_dpo/margin_mean": -0.011333346366882324, + "margin_dpo/margin_std": 0.312725692987442, + "step": 26 + }, + { + "KL/chosen_KL_mean": -0.008703231811523438, + "KL/mean": 0.0004301518201828003, + "KL/rejected_KL_mean": 0.009555816650390625, + "KL/std": 0.20847353339195251, + "epoch": 0.04081632653061224, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01825806498527527, + "fcm_dpo/q_t": 0.5004564523696899, + "grad_norm": 29.061359405517578, + "learning_rate": 1.9402985074626865e-07, + "logits/chosen": 0.08432801812887192, + "logits/rejected": 0.06020671874284744, + "logps/chosen": -62.88572692871094, + "logps/ref_chosen": -62.87702560424805, + "logps/ref_rejected": -71.34437561035156, + "logps/rejected": -71.33482360839844, + "loss": 1.3883, + "margin_dpo/margin_mean": -0.018257856369018555, + "margin_dpo/margin_std": 0.2578927278518677, + "step": 27 + }, + { + "KL/chosen_KL_mean": -0.009159088134765625, + "KL/mean": -0.031014487147331238, + "KL/rejected_KL_mean": -0.05286979675292969, + "KL/std": 0.21028277277946472, + "epoch": 0.042328042328042326, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.043710678815841675, + "fcm_dpo/q_t": 0.49890759587287903, + "grad_norm": 27.574337005615234, + "learning_rate": 2.0149253731343282e-07, + "logits/chosen": 0.055173229426145554, + "logits/rejected": 0.0465172678232193, + "logps/chosen": -59.84253692626953, + "logps/ref_chosen": -59.8333740234375, + "logps/ref_rejected": -70.39804077148438, + "logps/rejected": -70.4509048461914, + "loss": 1.3821, + "margin_dpo/margin_mean": 0.043710947036743164, + "margin_dpo/margin_std": 0.2772536277770996, + "step": 28 + }, + { + "KL/chosen_KL_mean": -0.013187408447265625, + "KL/mean": -0.05461312085390091, + "KL/rejected_KL_mean": -0.096038818359375, + "KL/std": 0.2508997321128845, + "epoch": 0.04383975812547241, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0828552097082138, + "fcm_dpo/q_t": 0.49792876839637756, + "grad_norm": 32.557945251464844, + "learning_rate": 2.08955223880597e-07, + "logits/chosen": 0.14180772006511688, + "logits/rejected": 0.1240900456905365, + "logps/chosen": -74.13339233398438, + "logps/ref_chosen": -74.12020111083984, + "logps/ref_rejected": -83.33099365234375, + "logps/rejected": -83.42703247070312, + "loss": 1.3783, + "margin_dpo/margin_mean": 0.08285492658615112, + "margin_dpo/margin_std": 0.3496696352958679, + "step": 29 + }, + { + "KL/chosen_KL_mean": -0.027311325073242188, + "KL/mean": -0.028391048312187195, + "KL/rejected_KL_mean": -0.02947235107421875, + "KL/std": 0.2199687361717224, + "epoch": 0.045351473922902494, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.0021583735942840576, + "fcm_dpo/q_t": 0.4999462962150574, + "grad_norm": 30.37353515625, + "learning_rate": 2.1641791044776117e-07, + "logits/chosen": 0.1467725932598114, + "logits/rejected": 0.09057148545980453, + "logps/chosen": -50.778602600097656, + "logps/ref_chosen": -50.75128936767578, + "logps/ref_rejected": -89.29063415527344, + "logps/rejected": -89.32010650634766, + "loss": 1.3863, + "margin_dpo/margin_mean": 0.0021587610244750977, + "margin_dpo/margin_std": 0.29320091009140015, + "step": 30 + }, + { + "KL/chosen_KL_mean": -0.019956588745117188, + "KL/mean": -0.015981465578079224, + "KL/rejected_KL_mean": -0.0120086669921875, + "KL/std": 0.2640872001647949, + "epoch": 0.04686318972033258, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.007950425148010254, + "fcm_dpo/q_t": 0.5001986622810364, + "grad_norm": 34.43701171875, + "learning_rate": 2.2388059701492537e-07, + "logits/chosen": 0.1202300488948822, + "logits/rejected": 0.07316860556602478, + "logps/chosen": -65.3567123413086, + "logps/ref_chosen": -65.33675384521484, + "logps/ref_rejected": -100.76666259765625, + "logps/rejected": -100.77867126464844, + "loss": 1.3874, + "margin_dpo/margin_mean": -0.007950246334075928, + "margin_dpo/margin_std": 0.3548576831817627, + "step": 31 + }, + { + "KL/chosen_KL_mean": 0.029937744140625, + "KL/mean": -0.003968536853790283, + "KL/rejected_KL_mean": -0.037876129150390625, + "KL/std": 0.21477648615837097, + "epoch": 0.04837490551776266, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06781303882598877, + "fcm_dpo/q_t": 0.498305082321167, + "grad_norm": 30.066669464111328, + "learning_rate": 2.3134328358208954e-07, + "logits/chosen": 0.07436567544937134, + "logits/rejected": 0.06658739596605301, + "logps/chosen": -67.15339660644531, + "logps/ref_chosen": -67.18333435058594, + "logps/ref_rejected": -82.80763244628906, + "logps/rejected": -82.84551239013672, + "loss": 1.3798, + "margin_dpo/margin_mean": 0.0678132176399231, + "margin_dpo/margin_std": 0.30971041321754456, + "step": 32 + }, + { + "KL/chosen_KL_mean": 0.0059452056884765625, + "KL/mean": -0.03278101235628128, + "KL/rejected_KL_mean": -0.071502685546875, + "KL/std": 0.2585922181606293, + "epoch": 0.049886621315192746, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07744117081165314, + "fcm_dpo/q_t": 0.49806421995162964, + "grad_norm": 30.893901824951172, + "learning_rate": 2.388059701492537e-07, + "logits/chosen": 0.02345607802271843, + "logits/rejected": -0.0022289585322141647, + "logps/chosen": -64.03353881835938, + "logps/ref_chosen": -64.03948211669922, + "logps/ref_rejected": -75.68357849121094, + "logps/rejected": -75.75508117675781, + "loss": 1.3789, + "margin_dpo/margin_mean": 0.07744191586971283, + "margin_dpo/margin_std": 0.35827744007110596, + "step": 33 + }, + { + "KL/chosen_KL_mean": -0.054653167724609375, + "KL/mean": -0.07489404082298279, + "KL/rejected_KL_mean": -0.09513473510742188, + "KL/std": 0.22619804739952087, + "epoch": 0.05139833711262283, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.040484070777893066, + "fcm_dpo/q_t": 0.4989892244338989, + "grad_norm": 28.486379623413086, + "learning_rate": 2.4626865671641786e-07, + "logits/chosen": 0.11268698424100876, + "logits/rejected": 0.08228495717048645, + "logps/chosen": -53.71894836425781, + "logps/ref_chosen": -53.6642951965332, + "logps/ref_rejected": -65.77989959716797, + "logps/rejected": -65.87503051757812, + "loss": 1.3825, + "margin_dpo/margin_mean": 0.04048377275466919, + "margin_dpo/margin_std": 0.3156694769859314, + "step": 34 + }, + { + "KL/chosen_KL_mean": -0.08617210388183594, + "KL/mean": -0.08068729937076569, + "KL/rejected_KL_mean": -0.07519912719726562, + "KL/std": 0.2363186478614807, + "epoch": 0.05291005291005291, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.01097145676612854, + "fcm_dpo/q_t": 0.5002747774124146, + "grad_norm": 27.84848403930664, + "learning_rate": 2.537313432835821e-07, + "logits/chosen": 0.041430506855249405, + "logits/rejected": 0.019430458545684814, + "logps/chosen": -61.10303497314453, + "logps/ref_chosen": -61.01686096191406, + "logps/ref_rejected": -72.78598022460938, + "logps/rejected": -72.8611831665039, + "loss": 1.3877, + "margin_dpo/margin_mean": -0.010971546173095703, + "margin_dpo/margin_std": 0.3413686752319336, + "step": 35 + }, + { + "KL/chosen_KL_mean": -0.07715034484863281, + "KL/mean": -0.08038248121738434, + "KL/rejected_KL_mean": -0.08361053466796875, + "KL/std": 0.21445265412330627, + "epoch": 0.05442176870748299, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.00646170973777771, + "fcm_dpo/q_t": 0.49983832240104675, + "grad_norm": 28.838197708129883, + "learning_rate": 2.611940298507462e-07, + "logits/chosen": 0.1054905503988266, + "logits/rejected": 0.05216691270470619, + "logps/chosen": -50.61450958251953, + "logps/ref_chosen": -50.53736114501953, + "logps/ref_rejected": -78.11678314208984, + "logps/rejected": -78.20039367675781, + "loss": 1.3859, + "margin_dpo/margin_mean": 0.006462037563323975, + "margin_dpo/margin_std": 0.3040819466114044, + "step": 36 + }, + { + "KL/chosen_KL_mean": -0.0345611572265625, + "KL/mean": -0.08462154865264893, + "KL/rejected_KL_mean": -0.13467788696289062, + "KL/std": 0.27500373125076294, + "epoch": 0.055933484504913075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1001129150390625, + "fcm_dpo/q_t": 0.4974990487098694, + "grad_norm": 37.900718688964844, + "learning_rate": 2.686567164179104e-07, + "logits/chosen": 0.1219300776720047, + "logits/rejected": 0.03991742804646492, + "logps/chosen": -59.58850860595703, + "logps/ref_chosen": -59.55394744873047, + "logps/ref_rejected": -108.27702331542969, + "logps/rejected": -108.41170501708984, + "loss": 1.3767, + "margin_dpo/margin_mean": 0.1001131534576416, + "margin_dpo/margin_std": 0.40394946932792664, + "step": 37 + }, + { + "KL/chosen_KL_mean": -0.061309814453125, + "KL/mean": -0.08432973921298981, + "KL/rejected_KL_mean": -0.10734939575195312, + "KL/std": 0.2561083137989044, + "epoch": 0.05744520030234316, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.046036750078201294, + "fcm_dpo/q_t": 0.49884986877441406, + "grad_norm": 29.617046356201172, + "learning_rate": 2.761194029850746e-07, + "logits/chosen": 0.07546387612819672, + "logits/rejected": 0.06145404651761055, + "logps/chosen": -65.84967041015625, + "logps/ref_chosen": -65.78836059570312, + "logps/ref_rejected": -76.1619873046875, + "logps/rejected": -76.26933288574219, + "loss": 1.382, + "margin_dpo/margin_mean": 0.046036988496780396, + "margin_dpo/margin_std": 0.34714776277542114, + "step": 38 + }, + { + "KL/chosen_KL_mean": -0.08732795715332031, + "KL/mean": -0.11383722722530365, + "KL/rejected_KL_mean": -0.14034271240234375, + "KL/std": 0.23883056640625, + "epoch": 0.05895691609977324, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.053016453981399536, + "fcm_dpo/q_t": 0.4986741840839386, + "grad_norm": 29.228801727294922, + "learning_rate": 2.8358208955223876e-07, + "logits/chosen": 0.15673673152923584, + "logits/rejected": 0.13004568219184875, + "logps/chosen": -57.264137268066406, + "logps/ref_chosen": -57.17681121826172, + "logps/ref_rejected": -79.486328125, + "logps/rejected": -79.62667846679688, + "loss": 1.3813, + "margin_dpo/margin_mean": 0.05301666259765625, + "margin_dpo/margin_std": 0.3345356583595276, + "step": 39 + }, + { + "KL/chosen_KL_mean": -0.09734535217285156, + "KL/mean": -0.11321689188480377, + "KL/rejected_KL_mean": -0.12909317016601562, + "KL/std": 0.2441941797733307, + "epoch": 0.06046863189720333, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.03174218535423279, + "fcm_dpo/q_t": 0.4992062747478485, + "grad_norm": 31.406536102294922, + "learning_rate": 2.9104477611940296e-07, + "logits/chosen": 0.10520674288272858, + "logits/rejected": 0.05589011684060097, + "logps/chosen": -61.43151092529297, + "logps/ref_chosen": -61.33416748046875, + "logps/ref_rejected": -79.10697174072266, + "logps/rejected": -79.23606872558594, + "loss": 1.3834, + "margin_dpo/margin_mean": 0.0317419171333313, + "margin_dpo/margin_std": 0.31017082929611206, + "step": 40 + }, + { + "KL/chosen_KL_mean": -0.069488525390625, + "KL/mean": -0.14583027362823486, + "KL/rejected_KL_mean": -0.22216796875, + "KL/std": 0.27735573053359985, + "epoch": 0.06198034769463341, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15267449617385864, + "fcm_dpo/q_t": 0.49618494510650635, + "grad_norm": 30.297801971435547, + "learning_rate": 2.985074626865671e-07, + "logits/chosen": 0.027745552361011505, + "logits/rejected": 0.008329922333359718, + "logps/chosen": -67.61621856689453, + "logps/ref_chosen": -67.5467300415039, + "logps/ref_rejected": -83.87788391113281, + "logps/rejected": -84.10005187988281, + "loss": 1.3715, + "margin_dpo/margin_mean": 0.1526748239994049, + "margin_dpo/margin_std": 0.38186556100845337, + "step": 41 + }, + { + "KL/chosen_KL_mean": -0.1026611328125, + "KL/mean": -0.10164512693881989, + "KL/rejected_KL_mean": -0.1006317138671875, + "KL/std": 0.2719181180000305, + "epoch": 0.06349206349206349, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": -0.0020251572132110596, + "fcm_dpo/q_t": 0.5000509023666382, + "grad_norm": 29.375078201293945, + "learning_rate": 3.059701492537313e-07, + "logits/chosen": 0.03170817345380783, + "logits/rejected": 0.010780701413750648, + "logps/chosen": -61.36751937866211, + "logps/ref_chosen": -61.26485824584961, + "logps/ref_rejected": -76.3629150390625, + "logps/rejected": -76.46354675292969, + "loss": 1.3868, + "margin_dpo/margin_mean": -0.0020249485969543457, + "margin_dpo/margin_std": 0.33517786860466003, + "step": 42 + }, + { + "KL/chosen_KL_mean": -0.0951690673828125, + "KL/mean": -0.14311568439006805, + "KL/rejected_KL_mean": -0.19105911254882812, + "KL/std": 0.2724185287952423, + "epoch": 0.06500377928949358, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.09589454531669617, + "fcm_dpo/q_t": 0.4976029396057129, + "grad_norm": 34.50521469116211, + "learning_rate": 3.134328358208955e-07, + "logits/chosen": 0.09837889671325684, + "logits/rejected": 0.08710268139839172, + "logps/chosen": -71.9041976928711, + "logps/ref_chosen": -71.80902862548828, + "logps/ref_rejected": -81.12464141845703, + "logps/rejected": -81.31570434570312, + "loss": 1.377, + "margin_dpo/margin_mean": 0.09589436650276184, + "margin_dpo/margin_std": 0.3491245210170746, + "step": 43 + }, + { + "KL/chosen_KL_mean": -0.1458568572998047, + "KL/mean": -0.15031519532203674, + "KL/rejected_KL_mean": -0.15477752685546875, + "KL/std": 0.31500521302223206, + "epoch": 0.06651549508692366, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.008913159370422363, + "fcm_dpo/q_t": 0.499778151512146, + "grad_norm": 32.60441207885742, + "learning_rate": 3.2089552238805965e-07, + "logits/chosen": 0.0608576163649559, + "logits/rejected": 0.02989174611866474, + "logps/chosen": -66.6962890625, + "logps/ref_chosen": -66.55043029785156, + "logps/ref_rejected": -85.06198120117188, + "logps/rejected": -85.21675872802734, + "loss": 1.386, + "margin_dpo/margin_mean": 0.008912891149520874, + "margin_dpo/margin_std": 0.4773399829864502, + "step": 44 + }, + { + "KL/chosen_KL_mean": -0.15514755249023438, + "KL/mean": -0.18950971961021423, + "KL/rejected_KL_mean": -0.223876953125, + "KL/std": 0.29197782278060913, + "epoch": 0.06802721088435375, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.06873787939548492, + "fcm_dpo/q_t": 0.49828290939331055, + "grad_norm": 31.94992446899414, + "learning_rate": 3.2835820895522385e-07, + "logits/chosen": 0.11507556587457657, + "logits/rejected": 0.0626973807811737, + "logps/chosen": -62.39900207519531, + "logps/ref_chosen": -62.24385452270508, + "logps/ref_rejected": -92.96665954589844, + "logps/rejected": -93.19053649902344, + "loss": 1.3799, + "margin_dpo/margin_mean": 0.06873822212219238, + "margin_dpo/margin_std": 0.41001275181770325, + "step": 45 + }, + { + "KL/chosen_KL_mean": -0.12407493591308594, + "KL/mean": -0.1781177967786789, + "KL/rejected_KL_mean": -0.2321624755859375, + "KL/std": 0.2968146800994873, + "epoch": 0.06953892668178382, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10808208584785461, + "fcm_dpo/q_t": 0.4972989559173584, + "grad_norm": 31.29147720336914, + "learning_rate": 3.3582089552238805e-07, + "logits/chosen": 0.105899378657341, + "logits/rejected": 0.06094621866941452, + "logps/chosen": -61.62297821044922, + "logps/ref_chosen": -61.498905181884766, + "logps/ref_rejected": -78.91172790527344, + "logps/rejected": -79.14389038085938, + "loss": 1.3759, + "margin_dpo/margin_mean": 0.10808172821998596, + "margin_dpo/margin_std": 0.39205238223075867, + "step": 46 + }, + { + "KL/chosen_KL_mean": -0.11026382446289062, + "KL/mean": -0.20422601699829102, + "KL/rejected_KL_mean": -0.2981910705566406, + "KL/std": 0.2935364246368408, + "epoch": 0.0710506424792139, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.1879267692565918, + "fcm_dpo/q_t": 0.49530404806137085, + "grad_norm": 28.419795989990234, + "learning_rate": 3.432835820895522e-07, + "logits/chosen": 0.016331728547811508, + "logits/rejected": -0.025488249957561493, + "logps/chosen": -51.6886100769043, + "logps/ref_chosen": -51.578346252441406, + "logps/ref_rejected": -68.2215576171875, + "logps/rejected": -68.51974487304688, + "loss": 1.368, + "margin_dpo/margin_mean": 0.1879270374774933, + "margin_dpo/margin_std": 0.40920567512512207, + "step": 47 + }, + { + "KL/chosen_KL_mean": -0.2367420196533203, + "KL/mean": -0.25752562284469604, + "KL/rejected_KL_mean": -0.2783088684082031, + "KL/std": 0.3226526081562042, + "epoch": 0.07256235827664399, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04156145453453064, + "fcm_dpo/q_t": 0.4989626407623291, + "grad_norm": 26.815656661987305, + "learning_rate": 3.507462686567164e-07, + "logits/chosen": 0.1488482505083084, + "logits/rejected": 0.11908246576786041, + "logps/chosen": -52.0303955078125, + "logps/ref_chosen": -51.79365158081055, + "logps/ref_rejected": -64.22503662109375, + "logps/rejected": -64.50334930419922, + "loss": 1.3827, + "margin_dpo/margin_mean": 0.04156213998794556, + "margin_dpo/margin_std": 0.4701569080352783, + "step": 48 + }, + { + "KL/chosen_KL_mean": -0.21713638305664062, + "KL/mean": -0.2565869092941284, + "KL/rejected_KL_mean": -0.2960357666015625, + "KL/std": 0.3597581088542938, + "epoch": 0.07407407407407407, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.07889637351036072, + "fcm_dpo/q_t": 0.49802789092063904, + "grad_norm": 27.303890228271484, + "learning_rate": 3.5820895522388055e-07, + "logits/chosen": 0.03007693588733673, + "logits/rejected": 0.008644701912999153, + "logps/chosen": -58.35173797607422, + "logps/ref_chosen": -58.13460159301758, + "logps/ref_rejected": -64.63206481933594, + "logps/rejected": -64.9281005859375, + "loss": 1.379, + "margin_dpo/margin_mean": 0.07889679074287415, + "margin_dpo/margin_std": 0.47261208295822144, + "step": 49 + }, + { + "KL/chosen_KL_mean": -0.24511146545410156, + "KL/mean": -0.3320258855819702, + "KL/rejected_KL_mean": -0.4189414978027344, + "KL/std": 0.3327283263206482, + "epoch": 0.07558578987150416, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.17382773756980896, + "fcm_dpo/q_t": 0.4956565499305725, + "grad_norm": 27.868024826049805, + "learning_rate": 3.6567164179104475e-07, + "logits/chosen": 0.1038937121629715, + "logits/rejected": 0.07447810471057892, + "logps/chosen": -53.1015510559082, + "logps/ref_chosen": -52.85643768310547, + "logps/ref_rejected": -72.17460632324219, + "logps/rejected": -72.59354400634766, + "loss": 1.3693, + "margin_dpo/margin_mean": 0.17382755875587463, + "margin_dpo/margin_std": 0.3663654923439026, + "step": 50 + }, + { + "KL/chosen_KL_mean": -0.2926807403564453, + "KL/mean": -0.40134263038635254, + "KL/rejected_KL_mean": -0.5100059509277344, + "KL/std": 0.4317777454853058, + "epoch": 0.07709750566893424, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.21732762455940247, + "fcm_dpo/q_t": 0.4945792555809021, + "grad_norm": 30.74460220336914, + "learning_rate": 3.7313432835820895e-07, + "logits/chosen": 0.06967415660619736, + "logits/rejected": 0.042400989681482315, + "logps/chosen": -63.94912338256836, + "logps/ref_chosen": -63.65644073486328, + "logps/ref_rejected": -86.13229370117188, + "logps/rejected": -86.64230346679688, + "loss": 1.3657, + "margin_dpo/margin_mean": 0.21732813119888306, + "margin_dpo/margin_std": 0.6217765808105469, + "step": 51 + }, + { + "KL/chosen_KL_mean": -0.31897735595703125, + "KL/mean": -0.4159046709537506, + "KL/rejected_KL_mean": -0.5128288269042969, + "KL/std": 0.4179096221923828, + "epoch": 0.07860922146636433, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.19385020434856415, + "fcm_dpo/q_t": 0.4951585531234741, + "grad_norm": 32.662662506103516, + "learning_rate": 3.805970149253731e-07, + "logits/chosen": 0.10473719984292984, + "logits/rejected": 0.05381231755018234, + "logps/chosen": -68.15919494628906, + "logps/ref_chosen": -67.8402099609375, + "logps/ref_rejected": -96.97090911865234, + "logps/rejected": -97.48373413085938, + "loss": 1.3677, + "margin_dpo/margin_mean": 0.19385017454624176, + "margin_dpo/margin_std": 0.53850257396698, + "step": 52 + }, + { + "KL/chosen_KL_mean": -0.3017730712890625, + "KL/mean": -0.4041089117527008, + "KL/rejected_KL_mean": -0.5064449310302734, + "KL/std": 0.3949553966522217, + "epoch": 0.0801209372637944, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2046741247177124, + "fcm_dpo/q_t": 0.4948856830596924, + "grad_norm": 27.410791397094727, + "learning_rate": 3.880597014925373e-07, + "logits/chosen": 0.09911762923002243, + "logits/rejected": 0.08820095658302307, + "logps/chosen": -57.17991256713867, + "logps/ref_chosen": -56.87813949584961, + "logps/ref_rejected": -60.75569152832031, + "logps/rejected": -61.26213836669922, + "loss": 1.3665, + "margin_dpo/margin_mean": 0.20467454195022583, + "margin_dpo/margin_std": 0.4859734773635864, + "step": 53 + }, + { + "KL/chosen_KL_mean": -0.3710346221923828, + "KL/mean": -0.4490298628807068, + "KL/rejected_KL_mean": -0.5270233154296875, + "KL/std": 0.40610271692276, + "epoch": 0.08163265306122448, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.15598756074905396, + "fcm_dpo/q_t": 0.4961045980453491, + "grad_norm": 26.544723510742188, + "learning_rate": 3.9552238805970144e-07, + "logits/chosen": 0.05449896678328514, + "logits/rejected": 0.03929914906620979, + "logps/chosen": -47.63795471191406, + "logps/ref_chosen": -47.26692199707031, + "logps/ref_rejected": -62.19426727294922, + "logps/rejected": -62.721290588378906, + "loss": 1.3714, + "margin_dpo/margin_mean": 0.15598756074905396, + "margin_dpo/margin_std": 0.5128107070922852, + "step": 54 + }, + { + "KL/chosen_KL_mean": -0.37180328369140625, + "KL/mean": -0.502418041229248, + "KL/rejected_KL_mean": -0.6330375671386719, + "KL/std": 0.5039347410202026, + "epoch": 0.08314436885865457, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2612312436103821, + "fcm_dpo/q_t": 0.493480384349823, + "grad_norm": 31.368911743164062, + "learning_rate": 4.0298507462686564e-07, + "logits/chosen": 0.03797540068626404, + "logits/rejected": -0.038480669260025024, + "logps/chosen": -50.697994232177734, + "logps/ref_chosen": -50.32619094848633, + "logps/ref_rejected": -92.44389343261719, + "logps/rejected": -93.0769271850586, + "loss": 1.3615, + "margin_dpo/margin_mean": 0.26123106479644775, + "margin_dpo/margin_std": 0.6476048231124878, + "step": 55 + }, + { + "KL/chosen_KL_mean": -0.37983131408691406, + "KL/mean": -0.4809678792953491, + "KL/rejected_KL_mean": -0.5820980072021484, + "KL/std": 0.49334120750427246, + "epoch": 0.08465608465608465, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.20226767659187317, + "fcm_dpo/q_t": 0.4949469268321991, + "grad_norm": 27.56074333190918, + "learning_rate": 4.1044776119402984e-07, + "logits/chosen": 0.1390300840139389, + "logits/rejected": 0.11645844578742981, + "logps/chosen": -57.14680480957031, + "logps/ref_chosen": -56.766971588134766, + "logps/ref_rejected": -66.30504608154297, + "logps/rejected": -66.88714599609375, + "loss": 1.3672, + "margin_dpo/margin_mean": 0.20226797461509705, + "margin_dpo/margin_std": 0.6378985643386841, + "step": 56 + }, + { + "KL/chosen_KL_mean": -0.4938812255859375, + "KL/mean": -0.6576261520385742, + "KL/rejected_KL_mean": -0.8213768005371094, + "KL/std": 0.6081602573394775, + "epoch": 0.08616780045351474, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.32749098539352417, + "fcm_dpo/q_t": 0.4918261766433716, + "grad_norm": 30.68051528930664, + "learning_rate": 4.17910447761194e-07, + "logits/chosen": 0.11048568785190582, + "logits/rejected": 0.04570011422038078, + "logps/chosen": -58.261627197265625, + "logps/ref_chosen": -57.76774597167969, + "logps/ref_rejected": -82.75698852539062, + "logps/rejected": -83.578369140625, + "loss": 1.355, + "margin_dpo/margin_mean": 0.3274906277656555, + "margin_dpo/margin_std": 0.6616859436035156, + "step": 57 + }, + { + "KL/chosen_KL_mean": -0.5845069885253906, + "KL/mean": -0.6910281181335449, + "KL/rejected_KL_mean": -0.79754638671875, + "KL/std": 0.6407666206359863, + "epoch": 0.08767951625094482, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.21303868293762207, + "fcm_dpo/q_t": 0.4947340786457062, + "grad_norm": 30.33217430114746, + "learning_rate": 4.253731343283582e-07, + "logits/chosen": 0.0610976405441761, + "logits/rejected": 0.04559905827045441, + "logps/chosen": -73.34859466552734, + "logps/ref_chosen": -72.76408386230469, + "logps/ref_rejected": -84.49275207519531, + "logps/rejected": -85.29029846191406, + "loss": 1.3675, + "margin_dpo/margin_mean": 0.2130383849143982, + "margin_dpo/margin_std": 0.934725284576416, + "step": 58 + }, + { + "KL/chosen_KL_mean": -0.5385608673095703, + "KL/mean": -0.6846149563789368, + "KL/rejected_KL_mean": -0.8306694030761719, + "KL/std": 0.625579297542572, + "epoch": 0.08919123204837491, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.29210516810417175, + "fcm_dpo/q_t": 0.49272221326828003, + "grad_norm": 26.56612205505371, + "learning_rate": 4.3283582089552234e-07, + "logits/chosen": 0.141486257314682, + "logits/rejected": 0.07466916739940643, + "logps/chosen": -50.359336853027344, + "logps/ref_chosen": -49.820777893066406, + "logps/ref_rejected": -77.14368438720703, + "logps/rejected": -77.97434997558594, + "loss": 1.3592, + "margin_dpo/margin_mean": 0.29210513830184937, + "margin_dpo/margin_std": 0.8294851779937744, + "step": 59 + }, + { + "KL/chosen_KL_mean": -0.6194343566894531, + "KL/mean": -0.6478511691093445, + "KL/rejected_KL_mean": -0.6762733459472656, + "KL/std": 0.604761004447937, + "epoch": 0.09070294784580499, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.05683696269989014, + "fcm_dpo/q_t": 0.49857690930366516, + "grad_norm": 29.42184829711914, + "learning_rate": 4.4029850746268654e-07, + "logits/chosen": 0.13081349432468414, + "logits/rejected": 0.129398912191391, + "logps/chosen": -63.844207763671875, + "logps/ref_chosen": -63.22477340698242, + "logps/ref_rejected": -61.360477447509766, + "logps/rejected": -62.03675079345703, + "loss": 1.3822, + "margin_dpo/margin_mean": 0.056836992502212524, + "margin_dpo/margin_std": 0.7822847366333008, + "step": 60 + }, + { + "KL/chosen_KL_mean": -0.7384166717529297, + "KL/mean": -0.763372540473938, + "KL/rejected_KL_mean": -0.788330078125, + "KL/std": 0.7057664394378662, + "epoch": 0.09221466364323508, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.04991529881954193, + "fcm_dpo/q_t": 0.49874916672706604, + "grad_norm": 28.068857192993164, + "learning_rate": 4.4776119402985074e-07, + "logits/chosen": 0.1358620971441269, + "logits/rejected": 0.10343830287456512, + "logps/chosen": -49.75521469116211, + "logps/ref_chosen": -49.01679992675781, + "logps/ref_rejected": -74.90817260742188, + "logps/rejected": -75.69650268554688, + "loss": 1.3839, + "margin_dpo/margin_mean": 0.04991510510444641, + "margin_dpo/margin_std": 1.0057084560394287, + "step": 61 + }, + { + "KL/chosen_KL_mean": -0.7364215850830078, + "KL/mean": -0.8766722679138184, + "KL/rejected_KL_mean": -1.0169219970703125, + "KL/std": 0.6324623823165894, + "epoch": 0.09372637944066516, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2804996371269226, + "fcm_dpo/q_t": 0.49300244450569153, + "grad_norm": 28.45606231689453, + "learning_rate": 4.552238805970149e-07, + "logits/chosen": 0.09042011201381683, + "logits/rejected": 0.052184414118528366, + "logps/chosen": -63.48828887939453, + "logps/ref_chosen": -62.751869201660156, + "logps/ref_rejected": -78.93360900878906, + "logps/rejected": -79.95053100585938, + "loss": 1.3604, + "margin_dpo/margin_mean": 0.28049927949905396, + "margin_dpo/margin_std": 0.8676041960716248, + "step": 62 + }, + { + "KL/chosen_KL_mean": -0.5458240509033203, + "KL/mean": -0.8238434195518494, + "KL/rejected_KL_mean": -1.10186767578125, + "KL/std": 0.6494956612586975, + "epoch": 0.09523809523809523, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5560450553894043, + "fcm_dpo/q_t": 0.48612523078918457, + "grad_norm": 31.456323623657227, + "learning_rate": 4.626865671641791e-07, + "logits/chosen": 0.19269225001335144, + "logits/rejected": 0.16756196320056915, + "logps/chosen": -61.061073303222656, + "logps/ref_chosen": -60.51525115966797, + "logps/ref_rejected": -85.11021423339844, + "logps/rejected": -86.21208190917969, + "loss": 1.3329, + "margin_dpo/margin_mean": 0.5560446381568909, + "margin_dpo/margin_std": 0.7645876407623291, + "step": 63 + }, + { + "KL/chosen_KL_mean": -0.8115634918212891, + "KL/mean": -0.8741039037704468, + "KL/rejected_KL_mean": -0.9366493225097656, + "KL/std": 0.7203623056411743, + "epoch": 0.09674981103552532, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.12508702278137207, + "fcm_dpo/q_t": 0.49687549471855164, + "grad_norm": 26.05970573425293, + "learning_rate": 4.701492537313433e-07, + "logits/chosen": 0.09709125757217407, + "logits/rejected": 0.07198961824178696, + "logps/chosen": -52.018409729003906, + "logps/ref_chosen": -51.20684814453125, + "logps/ref_rejected": -66.93081665039062, + "logps/rejected": -67.86746978759766, + "loss": 1.3757, + "margin_dpo/margin_mean": 0.12508690357208252, + "margin_dpo/margin_std": 0.8541518449783325, + "step": 64 + }, + { + "KL/chosen_KL_mean": -0.8654499053955078, + "KL/mean": -1.119699478149414, + "KL/rejected_KL_mean": -1.3739585876464844, + "KL/std": 0.9102262258529663, + "epoch": 0.0982615268329554, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.5085126757621765, + "fcm_dpo/q_t": 0.4873509407043457, + "grad_norm": 30.81188201904297, + "learning_rate": 4.776119402985074e-07, + "logits/chosen": 0.17681574821472168, + "logits/rejected": 0.1477317214012146, + "logps/chosen": -68.15414428710938, + "logps/ref_chosen": -67.2886962890625, + "logps/ref_rejected": -74.44281005859375, + "logps/rejected": -75.81676483154297, + "loss": 1.3396, + "margin_dpo/margin_mean": 0.5085115432739258, + "margin_dpo/margin_std": 1.1923322677612305, + "step": 65 + }, + { + "KL/chosen_KL_mean": -0.9162693023681641, + "KL/mean": -1.089837908744812, + "KL/rejected_KL_mean": -1.2634048461914062, + "KL/std": 0.8555461764335632, + "epoch": 0.09977324263038549, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3471396565437317, + "fcm_dpo/q_t": 0.4913522005081177, + "grad_norm": 29.340669631958008, + "learning_rate": 4.850746268656717e-07, + "logits/chosen": 0.08247184008359909, + "logits/rejected": 0.0590934120118618, + "logps/chosen": -71.65968322753906, + "logps/ref_chosen": -70.743408203125, + "logps/ref_rejected": -77.26499938964844, + "logps/rejected": -78.52841186523438, + "loss": 1.3551, + "margin_dpo/margin_mean": 0.3471394181251526, + "margin_dpo/margin_std": 1.1083390712738037, + "step": 66 + }, + { + "KL/chosen_KL_mean": -0.7525310516357422, + "KL/mean": -0.9344923496246338, + "KL/rejected_KL_mean": -1.116455078125, + "KL/std": 0.7987834215164185, + "epoch": 0.10128495842781557, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.3639173209667206, + "fcm_dpo/q_t": 0.4909346401691437, + "grad_norm": 28.93231773376465, + "learning_rate": 4.925373134328357e-07, + "logits/chosen": 0.0941900908946991, + "logits/rejected": 0.038454070687294006, + "logps/chosen": -61.355133056640625, + "logps/ref_chosen": -60.60260009765625, + "logps/ref_rejected": -75.22235870361328, + "logps/rejected": -76.33881378173828, + "loss": 1.3526, + "margin_dpo/margin_mean": 0.36391735076904297, + "margin_dpo/margin_std": 0.9612286686897278, + "step": 67 + }, + { + "KL/chosen_KL_mean": -1.2781333923339844, + "KL/mean": -1.3989845514297485, + "KL/rejected_KL_mean": -1.5198402404785156, + "KL/std": 0.9294252395629883, + "epoch": 0.10279667422524566, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24170318245887756, + "fcm_dpo/q_t": 0.4939650297164917, + "grad_norm": 31.69757080078125, + "learning_rate": 5e-07, + "logits/chosen": 0.050860900431871414, + "logits/rejected": 0.021575551480054855, + "logps/chosen": -78.80650329589844, + "logps/ref_chosen": -77.52836608886719, + "logps/ref_rejected": -93.17778015136719, + "logps/rejected": -94.69761657714844, + "loss": 1.3659, + "margin_dpo/margin_mean": 0.24170315265655518, + "margin_dpo/margin_std": 1.1950879096984863, + "step": 68 + }, + { + "KL/chosen_KL_mean": -1.1919231414794922, + "KL/mean": -1.5034534931182861, + "KL/rejected_KL_mean": -1.8149757385253906, + "KL/std": 1.018207311630249, + "epoch": 0.10430839002267574, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6230592131614685, + "fcm_dpo/q_t": 0.48451846837997437, + "grad_norm": 30.83071517944336, + "learning_rate": 4.999965034812934e-07, + "logits/chosen": 0.10441286861896515, + "logits/rejected": 0.06127467751502991, + "logps/chosen": -67.13497924804688, + "logps/ref_chosen": -65.94305419921875, + "logps/ref_rejected": -89.7735595703125, + "logps/rejected": -91.58853149414062, + "loss": 1.329, + "margin_dpo/margin_mean": 0.6230591535568237, + "margin_dpo/margin_std": 1.241918683052063, + "step": 69 + }, + { + "KL/chosen_KL_mean": -1.2819061279296875, + "KL/mean": -1.426012635231018, + "KL/rejected_KL_mean": -1.570119857788086, + "KL/std": 0.9026957750320435, + "epoch": 0.10582010582010581, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.28820908069610596, + "fcm_dpo/q_t": 0.4928149878978729, + "grad_norm": 28.84040069580078, + "learning_rate": 4.999860140229787e-07, + "logits/chosen": 0.1607946753501892, + "logits/rejected": 0.13725364208221436, + "logps/chosen": -63.23981857299805, + "logps/ref_chosen": -61.95791244506836, + "logps/ref_rejected": -75.80945587158203, + "logps/rejected": -77.37957763671875, + "loss": 1.3615, + "margin_dpo/margin_mean": 0.28820937871932983, + "margin_dpo/margin_std": 1.233094573020935, + "step": 70 + }, + { + "KL/chosen_KL_mean": -1.4533882141113281, + "KL/mean": -1.5742793083190918, + "KL/rejected_KL_mean": -1.6951732635498047, + "KL/std": 0.9835730791091919, + "epoch": 0.1073318216175359, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.24178528785705566, + "fcm_dpo/q_t": 0.493956983089447, + "grad_norm": 28.50391387939453, + "learning_rate": 4.999685319184688e-07, + "logits/chosen": 0.11954666674137115, + "logits/rejected": 0.1033221036195755, + "logps/chosen": -64.80096435546875, + "logps/ref_chosen": -63.34757995605469, + "logps/ref_rejected": -67.49658203125, + "logps/rejected": -69.19175720214844, + "loss": 1.3672, + "margin_dpo/margin_mean": 0.24178549647331238, + "margin_dpo/margin_std": 1.401712417602539, + "step": 71 + }, + { + "KL/chosen_KL_mean": -1.1878604888916016, + "KL/mean": -1.5602757930755615, + "KL/rejected_KL_mean": -1.9326896667480469, + "KL/std": 1.0539242029190063, + "epoch": 0.10884353741496598, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7448288798332214, + "fcm_dpo/q_t": 0.48147574067115784, + "grad_norm": 30.208765029907227, + "learning_rate": 4.999440576567755e-07, + "logits/chosen": 0.12368470430374146, + "logits/rejected": 0.06012523174285889, + "logps/chosen": -57.047157287597656, + "logps/ref_chosen": -55.85929870605469, + "logps/ref_rejected": -68.45423889160156, + "logps/rejected": -70.38693237304688, + "loss": 1.3175, + "margin_dpo/margin_mean": 0.7448291778564453, + "margin_dpo/margin_std": 1.263599157333374, + "step": 72 + }, + { + "KL/chosen_KL_mean": -1.8136749267578125, + "KL/mean": -1.8670963048934937, + "KL/rejected_KL_mean": -1.9205188751220703, + "KL/std": 1.2097903490066528, + "epoch": 0.11035525321239607, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.10684701800346375, + "fcm_dpo/q_t": 0.49749088287353516, + "grad_norm": 32.17892074584961, + "learning_rate": 4.999125919224965e-07, + "logits/chosen": 0.10997403413057327, + "logits/rejected": 0.09589383006095886, + "logps/chosen": -70.95248413085938, + "logps/ref_chosen": -69.13880920410156, + "logps/ref_rejected": -79.04586791992188, + "logps/rejected": -80.96638488769531, + "loss": 1.3826, + "margin_dpo/margin_mean": 0.10684752464294434, + "margin_dpo/margin_std": 1.594315767288208, + "step": 73 + }, + { + "KL/chosen_KL_mean": -1.2394580841064453, + "KL/mean": -1.5940744876861572, + "KL/rejected_KL_mean": -1.9486885070800781, + "KL/std": 1.0074858665466309, + "epoch": 0.11186696900982615, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7092300653457642, + "fcm_dpo/q_t": 0.4823755621910095, + "grad_norm": 27.89491844177246, + "learning_rate": 4.998741355957963e-07, + "logits/chosen": 0.1398930549621582, + "logits/rejected": 0.08669412136077881, + "logps/chosen": -51.16319274902344, + "logps/ref_chosen": -49.923736572265625, + "logps/ref_rejected": -81.73213958740234, + "logps/rejected": -83.68083190917969, + "loss": 1.321, + "margin_dpo/margin_mean": 0.7092299461364746, + "margin_dpo/margin_std": 1.2977672815322876, + "step": 74 + }, + { + "KL/chosen_KL_mean": -1.3947696685791016, + "KL/mean": -1.774251103401184, + "KL/rejected_KL_mean": -2.153726577758789, + "KL/std": 1.0954103469848633, + "epoch": 0.11337868480725624, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7589582204818726, + "fcm_dpo/q_t": 0.4811512529850006, + "grad_norm": 26.196157455444336, + "learning_rate": 4.998286897523808e-07, + "logits/chosen": 0.11667068302631378, + "logits/rejected": 0.08428828418254852, + "logps/chosen": -47.463523864746094, + "logps/ref_chosen": -46.06875228881836, + "logps/ref_rejected": -66.1181411743164, + "logps/rejected": -68.27186584472656, + "loss": 1.3164, + "margin_dpo/margin_mean": 0.7589577436447144, + "margin_dpo/margin_std": 1.351873517036438, + "step": 75 + }, + { + "KL/chosen_KL_mean": -1.6196708679199219, + "KL/mean": -1.7686243057250977, + "KL/rejected_KL_mean": -1.9175758361816406, + "KL/std": 1.199947476387024, + "epoch": 0.11489040060468632, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.2979068160057068, + "fcm_dpo/q_t": 0.492590069770813, + "grad_norm": 29.43617820739746, + "learning_rate": 4.997762556634679e-07, + "logits/chosen": 0.13128617405891418, + "logits/rejected": 0.08711592108011246, + "logps/chosen": -55.68241882324219, + "logps/ref_chosen": -54.06275177001953, + "logps/ref_rejected": -74.87464141845703, + "logps/rejected": -76.79222106933594, + "loss": 1.3628, + "margin_dpo/margin_mean": 0.2979065179824829, + "margin_dpo/margin_std": 1.5639325380325317, + "step": 76 + }, + { + "KL/chosen_KL_mean": -1.6665496826171875, + "KL/mean": -2.0357329845428467, + "KL/rejected_KL_mean": -2.4049110412597656, + "KL/std": 1.248659610748291, + "epoch": 0.1164021164021164, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.7383638620376587, + "fcm_dpo/q_t": 0.48167288303375244, + "grad_norm": 29.258098602294922, + "learning_rate": 4.99716834795752e-07, + "logits/chosen": 0.14366444945335388, + "logits/rejected": 0.10307327657938004, + "logps/chosen": -54.74264144897461, + "logps/ref_chosen": -53.07609176635742, + "logps/ref_rejected": -74.45601654052734, + "logps/rejected": -76.86093139648438, + "loss": 1.3193, + "margin_dpo/margin_mean": 0.7383638620376587, + "margin_dpo/margin_std": 1.4465680122375488, + "step": 77 + }, + { + "KL/chosen_KL_mean": -1.8012771606445312, + "KL/mean": -2.046334981918335, + "KL/rejected_KL_mean": -2.291393280029297, + "KL/std": 1.2755669355392456, + "epoch": 0.11791383219954649, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.49010640382766724, + "fcm_dpo/q_t": 0.4878043830394745, + "grad_norm": 29.197647094726562, + "learning_rate": 4.996504288113623e-07, + "logits/chosen": 0.1221899539232254, + "logits/rejected": 0.10165031254291534, + "logps/chosen": -69.52670288085938, + "logps/ref_chosen": -67.72541809082031, + "logps/ref_rejected": -79.03926849365234, + "logps/rejected": -81.33065795898438, + "loss": 1.3427, + "margin_dpo/margin_mean": 0.4901065230369568, + "margin_dpo/margin_std": 1.3799508810043335, + "step": 78 + }, + { + "KL/chosen_KL_mean": -1.7325191497802734, + "KL/mean": -2.309401035308838, + "KL/rejected_KL_mean": -2.886280059814453, + "KL/std": 1.5585962533950806, + "epoch": 0.11942554799697656, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1537659168243408, + "fcm_dpo/q_t": 0.471696138381958, + "grad_norm": 30.854867935180664, + "learning_rate": 4.995770395678171e-07, + "logits/chosen": 0.1881328523159027, + "logits/rejected": 0.12662355601787567, + "logps/chosen": -53.893165588378906, + "logps/ref_chosen": -52.16064453125, + "logps/ref_rejected": -83.31062316894531, + "logps/rejected": -86.1968994140625, + "loss": 1.2842, + "margin_dpo/margin_mean": 1.1537657976150513, + "margin_dpo/margin_std": 2.0103793144226074, + "step": 79 + }, + { + "KL/chosen_KL_mean": -2.034923553466797, + "KL/mean": -2.327785015106201, + "KL/rejected_KL_mean": -2.620647430419922, + "KL/std": 1.3555982112884521, + "epoch": 0.12093726379440665, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.585728645324707, + "fcm_dpo/q_t": 0.4855394661426544, + "grad_norm": 28.42696762084961, + "learning_rate": 4.994966691179711e-07, + "logits/chosen": 0.18274253606796265, + "logits/rejected": 0.12001022696495056, + "logps/chosen": -63.44548416137695, + "logps/ref_chosen": -61.410560607910156, + "logps/ref_rejected": -78.66004943847656, + "logps/rejected": -81.28070068359375, + "loss": 1.3382, + "margin_dpo/margin_mean": 0.5857288837432861, + "margin_dpo/margin_std": 1.8908803462982178, + "step": 80 + }, + { + "KL/chosen_KL_mean": -2.1589279174804688, + "KL/mean": -2.583031415939331, + "KL/rejected_KL_mean": -3.0071258544921875, + "KL/std": 1.4303652048110962, + "epoch": 0.12244897959183673, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8482010364532471, + "fcm_dpo/q_t": 0.47908836603164673, + "grad_norm": 29.102340698242188, + "learning_rate": 4.994093197099587e-07, + "logits/chosen": 0.09255268424749374, + "logits/rejected": 0.060397159308195114, + "logps/chosen": -65.96330261230469, + "logps/ref_chosen": -63.80437088012695, + "logps/ref_rejected": -79.3484115600586, + "logps/rejected": -82.35553741455078, + "loss": 1.3112, + "margin_dpo/margin_mean": 0.8482010364532471, + "margin_dpo/margin_std": 1.7088102102279663, + "step": 81 + }, + { + "KL/chosen_KL_mean": -1.7687873840332031, + "KL/mean": -2.319984197616577, + "KL/rejected_KL_mean": -2.8711814880371094, + "KL/std": 1.361003041267395, + "epoch": 0.12396069538926682, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1023948192596436, + "fcm_dpo/q_t": 0.4726252555847168, + "grad_norm": 26.5628662109375, + "learning_rate": 4.993149937871306e-07, + "logits/chosen": 0.11707571893930435, + "logits/rejected": 0.05370712652802467, + "logps/chosen": -50.58667755126953, + "logps/ref_chosen": -48.817893981933594, + "logps/ref_rejected": -70.31497955322266, + "logps/rejected": -73.1861572265625, + "loss": 1.2853, + "margin_dpo/margin_mean": 1.102394700050354, + "margin_dpo/margin_std": 1.5565245151519775, + "step": 82 + }, + { + "KL/chosen_KL_mean": -2.0592575073242188, + "KL/mean": -2.6452550888061523, + "KL/rejected_KL_mean": -3.2312545776367188, + "KL/std": 1.4756579399108887, + "epoch": 0.1254724111866969, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.171998381614685, + "fcm_dpo/q_t": 0.47112199664115906, + "grad_norm": 29.073789596557617, + "learning_rate": 4.992136939879856e-07, + "logits/chosen": 0.20907548069953918, + "logits/rejected": 0.158308207988739, + "logps/chosen": -59.21002960205078, + "logps/ref_chosen": -57.15077209472656, + "logps/ref_rejected": -75.1710205078125, + "logps/rejected": -78.40228271484375, + "loss": 1.2813, + "margin_dpo/margin_mean": 1.1719980239868164, + "margin_dpo/margin_std": 1.870093584060669, + "step": 83 + }, + { + "KL/chosen_KL_mean": -2.5398941040039062, + "KL/mean": -2.9809305667877197, + "KL/rejected_KL_mean": -3.4219741821289062, + "KL/std": 1.8264161348342896, + "epoch": 0.12698412698412698, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8820803761482239, + "fcm_dpo/q_t": 0.4781956672668457, + "grad_norm": 30.656856536865234, + "learning_rate": 4.991054231460969e-07, + "logits/chosen": 0.16261720657348633, + "logits/rejected": 0.12106196582317352, + "logps/chosen": -67.31719970703125, + "logps/ref_chosen": -64.77729797363281, + "logps/ref_rejected": -84.71949768066406, + "logps/rejected": -88.14147186279297, + "loss": 1.3091, + "margin_dpo/margin_mean": 0.882080078125, + "margin_dpo/margin_std": 1.9064234495162964, + "step": 84 + }, + { + "KL/chosen_KL_mean": -2.2631759643554688, + "KL/mean": -2.8933534622192383, + "KL/rejected_KL_mean": -3.523529052734375, + "KL/std": 1.7980573177337646, + "epoch": 0.12849584278155707, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.260351300239563, + "fcm_dpo/q_t": 0.4689059555530548, + "grad_norm": 26.809600830078125, + "learning_rate": 4.989901842900325e-07, + "logits/chosen": 0.1798171103000641, + "logits/rejected": 0.13506868481636047, + "logps/chosen": -52.514869689941406, + "logps/ref_chosen": -50.25169372558594, + "logps/ref_rejected": -66.55439758300781, + "logps/rejected": -70.07792663574219, + "loss": 1.2749, + "margin_dpo/margin_mean": 1.2603509426116943, + "margin_dpo/margin_std": 2.012509822845459, + "step": 85 + }, + { + "KL/chosen_KL_mean": -2.834178924560547, + "KL/mean": -3.3251657485961914, + "KL/rejected_KL_mean": -3.8161544799804688, + "KL/std": 1.9246859550476074, + "epoch": 0.13000755857898716, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.981971025466919, + "fcm_dpo/q_t": 0.4757484197616577, + "grad_norm": 27.233366012573242, + "learning_rate": 4.988679806432711e-07, + "logits/chosen": 0.19833333790302277, + "logits/rejected": 0.179485023021698, + "logps/chosen": -63.563358306884766, + "logps/ref_chosen": -60.72917938232422, + "logps/ref_rejected": -72.30961608886719, + "logps/rejected": -76.12576293945312, + "loss": 1.3012, + "margin_dpo/margin_mean": 0.9819709658622742, + "margin_dpo/margin_std": 2.0746121406555176, + "step": 86 + }, + { + "KL/chosen_KL_mean": -3.0052719116210938, + "KL/mean": -3.545877695083618, + "KL/rejected_KL_mean": -4.08648681640625, + "KL/std": 2.072525978088379, + "epoch": 0.13151927437641722, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0812183618545532, + "fcm_dpo/q_t": 0.4733849763870239, + "grad_norm": 30.487892150878906, + "learning_rate": 4.987388156241114e-07, + "logits/chosen": 0.1579674482345581, + "logits/rejected": 0.09991664439439774, + "logps/chosen": -68.76324462890625, + "logps/ref_chosen": -65.75796508789062, + "logps/ref_rejected": -84.81159973144531, + "logps/rejected": -88.89808654785156, + "loss": 1.2966, + "margin_dpo/margin_mean": 1.0812183618545532, + "margin_dpo/margin_std": 2.4813146591186523, + "step": 87 + }, + { + "KL/chosen_KL_mean": -3.063535690307617, + "KL/mean": -3.476155996322632, + "KL/rejected_KL_mean": -3.8887786865234375, + "KL/std": 2.3003952503204346, + "epoch": 0.1330309901738473, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8252429962158203, + "fcm_dpo/q_t": 0.4798470735549927, + "grad_norm": 30.224220275878906, + "learning_rate": 4.986026928455767e-07, + "logits/chosen": 0.261371374130249, + "logits/rejected": 0.2336939126253128, + "logps/chosen": -65.88756561279297, + "logps/ref_chosen": -62.82402801513672, + "logps/ref_rejected": -74.9607162475586, + "logps/rejected": -78.84949493408203, + "loss": 1.3276, + "margin_dpo/margin_mean": 0.8252427577972412, + "margin_dpo/margin_std": 2.88051176071167, + "step": 88 + }, + { + "KL/chosen_KL_mean": -2.5912418365478516, + "KL/mean": -3.2650604248046875, + "KL/rejected_KL_mean": -3.9388771057128906, + "KL/std": 2.12445330619812, + "epoch": 0.1345427059712774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3476394414901733, + "fcm_dpo/q_t": 0.46714815497398376, + "grad_norm": 29.441646575927734, + "learning_rate": 4.984596161153135e-07, + "logits/chosen": 0.23951318860054016, + "logits/rejected": 0.15807291865348816, + "logps/chosen": -43.782676696777344, + "logps/ref_chosen": -41.191436767578125, + "logps/ref_rejected": -85.44769287109375, + "logps/rejected": -89.3865737915039, + "loss": 1.2722, + "margin_dpo/margin_mean": 1.347640037536621, + "margin_dpo/margin_std": 2.477191925048828, + "step": 89 + }, + { + "KL/chosen_KL_mean": -3.108419418334961, + "KL/mean": -3.7823739051818848, + "KL/rejected_KL_mean": -4.456329345703125, + "KL/std": 2.459463119506836, + "epoch": 0.1360544217687075, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3479089736938477, + "fcm_dpo/q_t": 0.46694666147232056, + "grad_norm": 29.042463302612305, + "learning_rate": 4.983095894354857e-07, + "logits/chosen": 0.17343808710575104, + "logits/rejected": 0.1167207658290863, + "logps/chosen": -59.69232940673828, + "logps/ref_chosen": -56.58390808105469, + "logps/ref_rejected": -86.86978149414062, + "logps/rejected": -91.32611083984375, + "loss": 1.2718, + "margin_dpo/margin_mean": 1.3479087352752686, + "margin_dpo/margin_std": 2.471687078475952, + "step": 90 + }, + { + "KL/chosen_KL_mean": -3.1440505981445312, + "KL/mean": -3.712189197540283, + "KL/rejected_KL_mean": -4.280326843261719, + "KL/std": 2.318143367767334, + "epoch": 0.13756613756613756, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.1362712383270264, + "fcm_dpo/q_t": 0.47258251905441284, + "grad_norm": 25.776365280151367, + "learning_rate": 4.98152617002662e-07, + "logits/chosen": 0.15495944023132324, + "logits/rejected": 0.1115202009677887, + "logps/chosen": -55.52639389038086, + "logps/ref_chosen": -52.38234329223633, + "logps/ref_rejected": -72.17642211914062, + "logps/rejected": -76.45674896240234, + "loss": 1.2957, + "margin_dpo/margin_mean": 1.1362707614898682, + "margin_dpo/margin_std": 2.8179492950439453, + "step": 91 + }, + { + "KL/chosen_KL_mean": -3.261486053466797, + "KL/mean": -4.035880088806152, + "KL/rejected_KL_mean": -4.810276031494141, + "KL/std": 2.6673712730407715, + "epoch": 0.13907785336356765, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.54878830909729, + "fcm_dpo/q_t": 0.46264082193374634, + "grad_norm": 27.076570510864258, + "learning_rate": 4.979887032076988e-07, + "logits/chosen": 0.19806165993213654, + "logits/rejected": 0.15919730067253113, + "logps/chosen": -56.27018737792969, + "logps/ref_chosen": -53.00870132446289, + "logps/ref_rejected": -79.77812957763672, + "logps/rejected": -84.58840942382812, + "loss": 1.262, + "margin_dpo/margin_mean": 1.548788070678711, + "margin_dpo/margin_std": 3.1535987854003906, + "step": 92 + }, + { + "KL/chosen_KL_mean": -3.5588035583496094, + "KL/mean": -4.07647180557251, + "KL/rejected_KL_mean": -4.594139099121094, + "KL/std": 2.6193394660949707, + "epoch": 0.14058956916099774, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.035336971282959, + "fcm_dpo/q_t": 0.47518739104270935, + "grad_norm": 24.457630157470703, + "learning_rate": 4.978178526356172e-07, + "logits/chosen": 0.19866763055324554, + "logits/rejected": 0.16968736052513123, + "logps/chosen": -48.465858459472656, + "logps/ref_chosen": -44.90705108642578, + "logps/ref_rejected": -58.7879524230957, + "logps/rejected": -63.3820915222168, + "loss": 1.3097, + "margin_dpo/margin_mean": 1.0353363752365112, + "margin_dpo/margin_std": 3.151601791381836, + "step": 93 + }, + { + "KL/chosen_KL_mean": -3.2767086029052734, + "KL/mean": -4.201934814453125, + "KL/rejected_KL_mean": -5.127159118652344, + "KL/std": 3.2154555320739746, + "epoch": 0.1421012849584278, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.850449562072754, + "fcm_dpo/q_t": 0.45555442571640015, + "grad_norm": 27.38306999206543, + "learning_rate": 4.976400700654751e-07, + "logits/chosen": 0.2603720426559448, + "logits/rejected": 0.21980708837509155, + "logps/chosen": -63.2144775390625, + "logps/ref_chosen": -59.93777084350586, + "logps/ref_rejected": -79.3138427734375, + "logps/rejected": -84.44099426269531, + "loss": 1.249, + "margin_dpo/margin_mean": 1.8504489660263062, + "margin_dpo/margin_std": 4.002280235290527, + "step": 94 + }, + { + "KL/chosen_KL_mean": -4.290737152099609, + "KL/mean": -5.077942371368408, + "KL/rejected_KL_mean": -5.865150451660156, + "KL/std": 2.722923755645752, + "epoch": 0.1436130007558579, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5744117498397827, + "fcm_dpo/q_t": 0.4619680643081665, + "grad_norm": 29.313270568847656, + "learning_rate": 4.974553604702332e-07, + "logits/chosen": 0.17224471271038055, + "logits/rejected": 0.10648466646671295, + "logps/chosen": -64.459228515625, + "logps/ref_chosen": -60.168487548828125, + "logps/ref_rejected": -90.73665618896484, + "logps/rejected": -96.601806640625, + "loss": 1.2648, + "margin_dpo/margin_mean": 1.574411392211914, + "margin_dpo/margin_std": 3.4756722450256348, + "step": 95 + }, + { + "KL/chosen_KL_mean": -4.17289924621582, + "KL/mean": -5.119402885437012, + "KL/rejected_KL_mean": -6.065910339355469, + "KL/std": 3.4462270736694336, + "epoch": 0.14512471655328799, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.8930134773254395, + "fcm_dpo/q_t": 0.4562452733516693, + "grad_norm": 27.278207778930664, + "learning_rate": 4.972637290166157e-07, + "logits/chosen": 0.20686191320419312, + "logits/rejected": 0.16340209543704987, + "logps/chosen": -64.8416748046875, + "logps/ref_chosen": -60.66877746582031, + "logps/ref_rejected": -88.30673217773438, + "logps/rejected": -94.37264251708984, + "loss": 1.2567, + "margin_dpo/margin_mean": 1.8930132389068604, + "margin_dpo/margin_std": 4.5411200523376465, + "step": 96 + }, + { + "KL/chosen_KL_mean": -5.011419296264648, + "KL/mean": -5.512674331665039, + "KL/rejected_KL_mean": -6.013935089111328, + "KL/std": 3.628410816192627, + "epoch": 0.14663643235071808, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.002521276473999, + "fcm_dpo/q_t": 0.47609376907348633, + "grad_norm": 36.306068420410156, + "learning_rate": 4.970651810649666e-07, + "logits/chosen": 0.12959648668766022, + "logits/rejected": 0.08636681735515594, + "logps/chosen": -70.0555419921875, + "logps/ref_chosen": -65.04412078857422, + "logps/ref_rejected": -78.42092895507812, + "logps/rejected": -84.43486022949219, + "loss": 1.3488, + "margin_dpo/margin_mean": 1.0025215148925781, + "margin_dpo/margin_std": 4.992741584777832, + "step": 97 + }, + { + "KL/chosen_KL_mean": -4.639884948730469, + "KL/mean": -4.984724521636963, + "KL/rejected_KL_mean": -5.329566955566406, + "KL/std": 3.2658982276916504, + "epoch": 0.14814814814814814, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.6896781325340271, + "fcm_dpo/q_t": 0.48278307914733887, + "grad_norm": 30.303451538085938, + "learning_rate": 4.968597221690985e-07, + "logits/chosen": 0.22047525644302368, + "logits/rejected": 0.19347813725471497, + "logps/chosen": -60.14311599731445, + "logps/ref_chosen": -55.503231048583984, + "logps/ref_rejected": -72.81553649902344, + "logps/rejected": -78.14511108398438, + "loss": 1.345, + "margin_dpo/margin_mean": 0.6896780729293823, + "margin_dpo/margin_std": 3.231722831726074, + "step": 98 + }, + { + "KL/chosen_KL_mean": -4.664216995239258, + "KL/mean": -5.174734592437744, + "KL/rejected_KL_mean": -5.685253143310547, + "KL/std": 3.4879510402679443, + "epoch": 0.14965986394557823, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.0210323333740234, + "fcm_dpo/q_t": 0.4757738709449768, + "grad_norm": 32.484554290771484, + "learning_rate": 4.966473580761389e-07, + "logits/chosen": 0.22554650902748108, + "logits/rejected": 0.18714788556098938, + "logps/chosen": -63.23985290527344, + "logps/ref_chosen": -58.57563781738281, + "logps/ref_rejected": -78.693603515625, + "logps/rejected": -84.37886047363281, + "loss": 1.3373, + "margin_dpo/margin_mean": 1.0210322141647339, + "margin_dpo/margin_std": 4.534329414367676, + "step": 99 + }, + { + "KL/chosen_KL_mean": -5.333049774169922, + "KL/mean": -6.067066669464111, + "KL/rejected_KL_mean": -6.801078796386719, + "KL/std": 4.175366401672363, + "epoch": 0.15117157974300832, + "fcm_dpo/beta": 0.10000000149011612, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.4680317640304565, + "fcm_dpo/q_t": 0.4672295153141022, + "grad_norm": 33.170841217041016, + "learning_rate": 4.964280947263676e-07, + "logits/chosen": 0.25199437141418457, + "logits/rejected": 0.24380481243133545, + "logps/chosen": -84.91648864746094, + "logps/ref_chosen": -79.58343505859375, + "logps/ref_rejected": -92.152587890625, + "logps/rejected": -98.95365905761719, + "loss": 1.3323, + "margin_dpo/margin_mean": 1.468031883239746, + "margin_dpo/margin_std": 5.88205623626709, + "step": 100 + }, + { + "KL/chosen_KL_mean": -4.298732757568359, + "KL/mean": -5.323648452758789, + "KL/rejected_KL_mean": -6.348566055297852, + "KL/std": 3.6733579635620117, + "epoch": 0.15268329554043839, + "fcm_dpo/beta": 0.10008342564105988, + "fcm_dpo/delta": 0.08335614204406738, + "fcm_dpo/margin": 2.049832820892334, + "fcm_dpo/q_t": 0.45039981603622437, + "grad_norm": 25.92251205444336, + "learning_rate": 4.96201938253052e-07, + "logits/chosen": 0.23845641314983368, + "logits/rejected": 0.1990615427494049, + "logps/chosen": -56.63151931762695, + "logps/ref_chosen": -52.332786560058594, + "logps/ref_rejected": -69.55589294433594, + "logps/rejected": -75.90446472167969, + "loss": 1.2356, + "margin_dpo/margin_mean": 2.049832820892334, + "margin_dpo/margin_std": 4.183335781097412, + "step": 101 + }, + { + "KL/chosen_KL_mean": -5.354887008666992, + "KL/mean": -5.76564884185791, + "KL/rejected_KL_mean": -6.176418304443359, + "KL/std": 3.713839292526245, + "epoch": 0.15419501133786848, + "fcm_dpo/beta": 0.10016684979200363, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.821533203125, + "fcm_dpo/q_t": 0.48328104615211487, + "grad_norm": 32.9635124206543, + "learning_rate": 4.959688949822748e-07, + "logits/chosen": 0.17115306854248047, + "logits/rejected": 0.13199609518051147, + "logps/chosen": -70.09837341308594, + "logps/ref_chosen": -64.74348449707031, + "logps/ref_rejected": -69.06132507324219, + "logps/rejected": -75.23774719238281, + "loss": 1.3777, + "margin_dpo/margin_mean": 0.8215329051017761, + "margin_dpo/margin_std": 5.5303239822387695, + "step": 102 + }, + { + "KL/chosen_KL_mean": -4.852344512939453, + "KL/mean": -5.623050689697266, + "KL/rejected_KL_mean": -6.393760681152344, + "KL/std": 3.946211576461792, + "epoch": 0.15570672713529857, + "fcm_dpo/beta": 0.10016684979200363, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5414180755615234, + "fcm_dpo/q_t": 0.46298670768737793, + "grad_norm": 29.602161407470703, + "learning_rate": 4.957289714327572e-07, + "logits/chosen": 0.26536405086517334, + "logits/rejected": 0.23137174546718597, + "logps/chosen": -68.6889877319336, + "logps/ref_chosen": -63.83664321899414, + "logps/ref_rejected": -79.32362365722656, + "logps/rejected": -85.71739196777344, + "loss": 1.2718, + "margin_dpo/margin_mean": 1.5414175987243652, + "margin_dpo/margin_std": 3.6745362281799316, + "step": 103 + }, + { + "KL/chosen_KL_mean": -4.706945419311523, + "KL/mean": -5.840055465698242, + "KL/rejected_KL_mean": -6.973175048828125, + "KL/std": 4.968792915344238, + "epoch": 0.15721844293272866, + "fcm_dpo/beta": 0.10023517906665802, + "fcm_dpo/delta": 0.06816459447145462, + "fcm_dpo/margin": 2.2662224769592285, + "fcm_dpo/q_t": 0.4515298306941986, + "grad_norm": 31.129642486572266, + "learning_rate": 4.954821743156767e-07, + "logits/chosen": 0.27063775062561035, + "logits/rejected": 0.18173637986183167, + "logps/chosen": -65.7061538696289, + "logps/ref_chosen": -60.99920654296875, + "logps/ref_rejected": -98.84645080566406, + "logps/rejected": -105.81962585449219, + "loss": 1.266, + "margin_dpo/margin_mean": 2.2662224769592285, + "margin_dpo/margin_std": 6.337430477142334, + "step": 104 + }, + { + "KL/chosen_KL_mean": -5.383846282958984, + "KL/mean": -6.280284404754639, + "KL/rejected_KL_mean": -7.176723480224609, + "KL/std": 4.030929088592529, + "epoch": 0.15873015873015872, + "fcm_dpo/beta": 0.10030350089073181, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.792877197265625, + "fcm_dpo/q_t": 0.45901936292648315, + "grad_norm": 31.023033142089844, + "learning_rate": 4.952285105344791e-07, + "logits/chosen": 0.22098499536514282, + "logits/rejected": 0.16488680243492126, + "logps/chosen": -76.33411407470703, + "logps/ref_chosen": -70.95027160644531, + "logps/ref_rejected": -87.88340759277344, + "logps/rejected": -95.06013488769531, + "loss": 1.2755, + "margin_dpo/margin_mean": 1.7928767204284668, + "margin_dpo/margin_std": 5.000432014465332, + "step": 105 + }, + { + "KL/chosen_KL_mean": -5.034534454345703, + "KL/mean": -5.834381103515625, + "KL/rejected_KL_mean": -6.6342315673828125, + "KL/std": 3.937655448913574, + "epoch": 0.1602418745275888, + "fcm_dpo/beta": 0.10030350089073181, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.5996934175491333, + "fcm_dpo/q_t": 0.461678683757782, + "grad_norm": 31.038833618164062, + "learning_rate": 4.949679871846857e-07, + "logits/chosen": 0.24851784110069275, + "logits/rejected": 0.2343132048845291, + "logps/chosen": -67.49386596679688, + "logps/ref_chosen": -62.45933151245117, + "logps/ref_rejected": -67.00595092773438, + "logps/rejected": -73.64018249511719, + "loss": 1.3007, + "margin_dpo/margin_mean": 1.5996935367584229, + "margin_dpo/margin_std": 5.308818817138672, + "step": 106 + }, + { + "KL/chosen_KL_mean": -6.537326812744141, + "KL/mean": -6.974967956542969, + "KL/rejected_KL_mean": -7.412609100341797, + "KL/std": 4.93362283706665, + "epoch": 0.1617535903250189, + "fcm_dpo/beta": 0.10030350089073181, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.8752791881561279, + "fcm_dpo/q_t": 0.47764110565185547, + "grad_norm": 41.651710510253906, + "learning_rate": 4.947006115536947e-07, + "logits/chosen": 0.17929807305335999, + "logits/rejected": 0.1554069221019745, + "logps/chosen": -82.37528991699219, + "logps/ref_chosen": -75.83796691894531, + "logps/ref_rejected": -87.74038696289062, + "logps/rejected": -95.15299224853516, + "loss": 1.3917, + "margin_dpo/margin_mean": 0.8752791881561279, + "margin_dpo/margin_std": 6.128002166748047, + "step": 107 + }, + { + "KL/chosen_KL_mean": -5.530330657958984, + "KL/mean": -6.4383392333984375, + "KL/rejected_KL_mean": -7.346343994140625, + "KL/std": 4.103663921356201, + "epoch": 0.16326530612244897, + "fcm_dpo/beta": 0.10030350089073181, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.816009521484375, + "fcm_dpo/q_t": 0.45837894082069397, + "grad_norm": 28.870685577392578, + "learning_rate": 4.944263911205772e-07, + "logits/chosen": 0.15603496134281158, + "logits/rejected": 0.12610435485839844, + "logps/chosen": -73.92356872558594, + "logps/ref_chosen": -68.39323425292969, + "logps/ref_rejected": -83.24267578125, + "logps/rejected": -90.58901977539062, + "loss": 1.2818, + "margin_dpo/margin_mean": 1.816009759902954, + "margin_dpo/margin_std": 5.3321051597595215, + "step": 108 + }, + { + "KL/chosen_KL_mean": -4.887001037597656, + "KL/mean": -6.407367706298828, + "KL/rejected_KL_mean": -7.927738189697266, + "KL/std": 5.069875717163086, + "epoch": 0.16477702191987906, + "fcm_dpo/beta": 0.10044832527637482, + "fcm_dpo/delta": 0.09485046565532684, + "fcm_dpo/margin": 3.0407395362854004, + "fcm_dpo/q_t": 0.43271785974502563, + "grad_norm": 28.4634952545166, + "learning_rate": 4.941453335558681e-07, + "logits/chosen": 0.22250492870807648, + "logits/rejected": 0.1653507649898529, + "logps/chosen": -60.41448211669922, + "logps/ref_chosen": -55.52748107910156, + "logps/ref_rejected": -83.55218505859375, + "logps/rejected": -91.47992706298828, + "loss": 1.1717, + "margin_dpo/margin_mean": 3.040739059448242, + "margin_dpo/margin_std": 5.4991679191589355, + "step": 109 + }, + { + "KL/chosen_KL_mean": -6.269718170166016, + "KL/mean": -6.398492336273193, + "KL/rejected_KL_mean": -6.527263641357422, + "KL/std": 4.076105117797852, + "epoch": 0.16628873771730915, + "fcm_dpo/beta": 0.10049396008253098, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 0.25754761695861816, + "fcm_dpo/q_t": 0.4914044737815857, + "grad_norm": 44.64370346069336, + "learning_rate": 4.938574467213517e-07, + "logits/chosen": 0.18484854698181152, + "logits/rejected": 0.19435881078243256, + "logps/chosen": -87.428466796875, + "logps/ref_chosen": -81.15874481201172, + "logps/ref_rejected": -72.56021118164062, + "logps/rejected": -79.08747863769531, + "loss": 1.4315, + "margin_dpo/margin_mean": 0.25754648447036743, + "margin_dpo/margin_std": 5.484380722045898, + "step": 110 + }, + { + "KL/chosen_KL_mean": -5.663007736206055, + "KL/mean": -6.669684410095215, + "KL/rejected_KL_mean": -7.676361083984375, + "KL/std": 4.254589080810547, + "epoch": 0.16780045351473924, + "fcm_dpo/beta": 0.10049396008253098, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.013352870941162, + "fcm_dpo/q_t": 0.4542198181152344, + "grad_norm": 25.791231155395508, + "learning_rate": 4.935627386698418e-07, + "logits/chosen": 0.2559598684310913, + "logits/rejected": 0.22001683712005615, + "logps/chosen": -58.02199172973633, + "logps/ref_chosen": -52.358985900878906, + "logps/ref_rejected": -77.06150817871094, + "logps/rejected": -84.73786926269531, + "loss": 1.2585, + "margin_dpo/margin_mean": 2.0133538246154785, + "margin_dpo/margin_std": 5.169063568115234, + "step": 111 + }, + { + "KL/chosen_KL_mean": -5.31280517578125, + "KL/mean": -6.554027557373047, + "KL/rejected_KL_mean": -7.795249938964844, + "KL/std": 4.280986785888672, + "epoch": 0.1693121693121693, + "fcm_dpo/beta": 0.10072842985391617, + "fcm_dpo/delta": 0.15031039714813232, + "fcm_dpo/margin": 2.482445240020752, + "fcm_dpo/q_t": 0.44088420271873474, + "grad_norm": 32.15288162231445, + "learning_rate": 4.932612176449559e-07, + "logits/chosen": 0.1908096969127655, + "logits/rejected": 0.12586526572704315, + "logps/chosen": -68.33287048339844, + "logps/ref_chosen": -63.02006530761719, + "logps/ref_rejected": -111.36941528320312, + "logps/rejected": -119.16466522216797, + "loss": 1.2074, + "margin_dpo/margin_mean": 2.4824447631835938, + "margin_dpo/margin_std": 4.781793117523193, + "step": 112 + }, + { + "KL/chosen_KL_mean": -5.411689758300781, + "KL/mean": -6.090734004974365, + "KL/rejected_KL_mean": -6.769779205322266, + "KL/std": 3.6462278366088867, + "epoch": 0.1708238851095994, + "fcm_dpo/beta": 0.10079652070999146, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.3580913543701172, + "fcm_dpo/q_t": 0.46707823872566223, + "grad_norm": 36.16310119628906, + "learning_rate": 4.929528920808854e-07, + "logits/chosen": 0.1987951248884201, + "logits/rejected": 0.15978413820266724, + "logps/chosen": -61.21935272216797, + "logps/ref_chosen": -55.80766296386719, + "logps/ref_rejected": -69.84014129638672, + "logps/rejected": -76.60992431640625, + "loss": 1.3083, + "margin_dpo/margin_mean": 1.358091115951538, + "margin_dpo/margin_std": 4.626516342163086, + "step": 113 + }, + { + "KL/chosen_KL_mean": -5.120885848999023, + "KL/mean": -6.2701945304870605, + "KL/rejected_KL_mean": -7.419506072998047, + "KL/std": 3.9188289642333984, + "epoch": 0.17233560090702948, + "fcm_dpo/beta": 0.10094878077507019, + "fcm_dpo/delta": 0.07546986639499664, + "fcm_dpo/margin": 2.2986130714416504, + "fcm_dpo/q_t": 0.4463236927986145, + "grad_norm": 29.769275665283203, + "learning_rate": 4.92637770602159e-07, + "logits/chosen": 0.2607458829879761, + "logits/rejected": 0.20008817315101624, + "logps/chosen": -71.45365905761719, + "logps/ref_chosen": -66.33277130126953, + "logps/ref_rejected": -71.61489868164062, + "logps/rejected": -79.0344009399414, + "loss": 1.2212, + "margin_dpo/margin_mean": 2.2986135482788086, + "margin_dpo/margin_std": 4.692882537841797, + "step": 114 + }, + { + "KL/chosen_KL_mean": -5.489625930786133, + "KL/mean": -6.440537929534912, + "KL/rejected_KL_mean": -7.391450881958008, + "KL/std": 3.5965399742126465, + "epoch": 0.17384731670445955, + "fcm_dpo/beta": 0.10113140940666199, + "fcm_dpo/delta": 0.09037352353334427, + "fcm_dpo/margin": 1.9018210172653198, + "fcm_dpo/q_t": 0.4549124240875244, + "grad_norm": 28.72669792175293, + "learning_rate": 4.923158620234019e-07, + "logits/chosen": 0.20971181988716125, + "logits/rejected": 0.1486668884754181, + "logps/chosen": -61.23866271972656, + "logps/ref_chosen": -55.74903869628906, + "logps/ref_rejected": -79.59849548339844, + "logps/rejected": -86.98994445800781, + "loss": 1.248, + "margin_dpo/margin_mean": 1.9018211364746094, + "margin_dpo/margin_std": 4.239630699157715, + "step": 115 + }, + { + "KL/chosen_KL_mean": -5.18193244934082, + "KL/mean": -6.607453346252441, + "KL/rejected_KL_mean": -8.032978057861328, + "KL/std": 3.699169635772705, + "epoch": 0.17535903250188964, + "fcm_dpo/beta": 0.10132897645235062, + "fcm_dpo/delta": 0.1113809123635292, + "fcm_dpo/margin": 2.8510475158691406, + "fcm_dpo/q_t": 0.43133842945098877, + "grad_norm": 26.183059692382812, + "learning_rate": 4.91987175349089e-07, + "logits/chosen": 0.23607221245765686, + "logits/rejected": 0.16755008697509766, + "logps/chosen": -54.547096252441406, + "logps/ref_chosen": -49.36516571044922, + "logps/ref_rejected": -72.84671020507812, + "logps/rejected": -80.87968444824219, + "loss": 1.1594, + "margin_dpo/margin_mean": 2.8510475158691406, + "margin_dpo/margin_std": 4.0363383293151855, + "step": 116 + }, + { + "KL/chosen_KL_mean": -5.154685974121094, + "KL/mean": -6.106015205383301, + "KL/rejected_KL_mean": -7.057350158691406, + "KL/std": 3.4758830070495605, + "epoch": 0.17687074829931973, + "fcm_dpo/beta": 0.1014070138335228, + "fcm_dpo/delta": 0.024697382003068924, + "fcm_dpo/margin": 1.9026578664779663, + "fcm_dpo/q_t": 0.45379531383514404, + "grad_norm": 28.551708221435547, + "learning_rate": 4.916517197732933e-07, + "logits/chosen": 0.25603365898132324, + "logits/rejected": 0.21851274371147156, + "logps/chosen": -62.86558532714844, + "logps/ref_chosen": -57.710899353027344, + "logps/ref_rejected": -69.77253723144531, + "logps/rejected": -76.82988739013672, + "loss": 1.2546, + "margin_dpo/margin_mean": 1.902658224105835, + "margin_dpo/margin_std": 4.315097808837891, + "step": 117 + }, + { + "KL/chosen_KL_mean": -5.071140289306641, + "KL/mean": -6.3113203048706055, + "KL/rejected_KL_mean": -7.551494598388672, + "KL/std": 3.793917179107666, + "epoch": 0.17838246409674982, + "fcm_dpo/beta": 0.10161709785461426, + "fcm_dpo/delta": 0.14834460616111755, + "fcm_dpo/margin": 2.4803643226623535, + "fcm_dpo/q_t": 0.4397216737270355, + "grad_norm": 28.526634216308594, + "learning_rate": 4.913095046794281e-07, + "logits/chosen": 0.2956094443798065, + "logits/rejected": 0.2542411684989929, + "logps/chosen": -57.55104064941406, + "logps/ref_chosen": -52.479896545410156, + "logps/ref_rejected": -81.359130859375, + "logps/rejected": -88.9106216430664, + "loss": 1.1887, + "margin_dpo/margin_mean": 2.4803640842437744, + "margin_dpo/margin_std": 3.8863167762756348, + "step": 118 + }, + { + "KL/chosen_KL_mean": -6.347293853759766, + "KL/mean": -7.282366752624512, + "KL/rejected_KL_mean": -8.217445373535156, + "KL/std": 4.4139556884765625, + "epoch": 0.17989417989417988, + "fcm_dpo/beta": 0.10179169476032257, + "fcm_dpo/delta": 0.08190931379795074, + "fcm_dpo/margin": 1.8701521158218384, + "fcm_dpo/q_t": 0.4565848410129547, + "grad_norm": 29.758865356445312, + "learning_rate": 4.909605396399855e-07, + "logits/chosen": 0.21873857080936432, + "logits/rejected": 0.17934158444404602, + "logps/chosen": -67.70497131347656, + "logps/ref_chosen": -61.35767364501953, + "logps/ref_rejected": -75.71510314941406, + "logps/rejected": -83.93254089355469, + "loss": 1.2778, + "margin_dpo/margin_mean": 1.870152235031128, + "margin_dpo/margin_std": 5.467679023742676, + "step": 119 + }, + { + "KL/chosen_KL_mean": -5.258893966674805, + "KL/mean": -6.6604413986206055, + "KL/rejected_KL_mean": -8.061992645263672, + "KL/std": 3.740701675415039, + "epoch": 0.18140589569160998, + "fcm_dpo/beta": 0.10206224024295807, + "fcm_dpo/delta": 0.11421863734722137, + "fcm_dpo/margin": 2.8031044006347656, + "fcm_dpo/q_t": 0.4322637915611267, + "grad_norm": 29.23235321044922, + "learning_rate": 4.906048344162676e-07, + "logits/chosen": 0.20290006697177887, + "logits/rejected": 0.143024742603302, + "logps/chosen": -65.16645812988281, + "logps/ref_chosen": -59.907569885253906, + "logps/ref_rejected": -79.6910629272461, + "logps/rejected": -87.7530517578125, + "loss": 1.1578, + "margin_dpo/margin_mean": 2.8031041622161865, + "margin_dpo/margin_std": 3.8060660362243652, + "step": 120 + }, + { + "KL/chosen_KL_mean": -5.834081649780273, + "KL/mean": -7.289121150970459, + "KL/rejected_KL_mean": -8.744163513183594, + "KL/std": 4.341035842895508, + "epoch": 0.18291761148904007, + "fcm_dpo/beta": 0.10225256532430649, + "fcm_dpo/delta": 0.10275236517190933, + "fcm_dpo/margin": 2.9100799560546875, + "fcm_dpo/q_t": 0.43305572867393494, + "grad_norm": 27.561906814575195, + "learning_rate": 4.902423989581143e-07, + "logits/chosen": 0.34082621335983276, + "logits/rejected": 0.2517605721950531, + "logps/chosen": -61.5001220703125, + "logps/ref_chosen": -55.66604232788086, + "logps/ref_rejected": -101.56233978271484, + "logps/rejected": -110.30650329589844, + "loss": 1.1669, + "margin_dpo/margin_mean": 2.9100794792175293, + "margin_dpo/margin_std": 4.866159915924072, + "step": 121 + }, + { + "KL/chosen_KL_mean": -6.321319580078125, + "KL/mean": -7.662225723266602, + "KL/rejected_KL_mean": -9.003135681152344, + "KL/std": 4.700608253479004, + "epoch": 0.18442932728647016, + "fcm_dpo/beta": 0.10253050178289413, + "fcm_dpo/delta": 0.12534500658512115, + "fcm_dpo/margin": 2.6818149089813232, + "fcm_dpo/q_t": 0.44021308422088623, + "grad_norm": 31.384111404418945, + "learning_rate": 4.898732434036243e-07, + "logits/chosen": 0.22849182784557343, + "logits/rejected": 0.19300708174705505, + "logps/chosen": -69.65569305419922, + "logps/ref_chosen": -63.334373474121094, + "logps/ref_rejected": -73.67523193359375, + "logps/rejected": -82.67837524414062, + "loss": 1.2072, + "margin_dpo/margin_mean": 2.681814670562744, + "margin_dpo/margin_std": 5.711152076721191, + "step": 122 + }, + { + "KL/chosen_KL_mean": -6.042579650878906, + "KL/mean": -7.319419860839844, + "KL/rejected_KL_mean": -8.596260070800781, + "KL/std": 4.097225189208984, + "epoch": 0.18594104308390022, + "fcm_dpo/beta": 0.10279840975999832, + "fcm_dpo/delta": 0.13782979547977448, + "fcm_dpo/margin": 2.553683042526245, + "fcm_dpo/q_t": 0.438998818397522, + "grad_norm": 27.8026065826416, + "learning_rate": 4.894973780788722e-07, + "logits/chosen": 0.2467765212059021, + "logits/rejected": 0.20531314611434937, + "logps/chosen": -62.941322326660156, + "logps/ref_chosen": -56.89874267578125, + "logps/ref_rejected": -78.97028350830078, + "logps/rejected": -87.56654357910156, + "loss": 1.1913, + "margin_dpo/margin_mean": 2.553682804107666, + "margin_dpo/margin_std": 4.479000091552734, + "step": 123 + }, + { + "KL/chosen_KL_mean": -6.389518737792969, + "KL/mean": -8.202165603637695, + "KL/rejected_KL_mean": -10.014816284179688, + "KL/std": 4.544312477111816, + "epoch": 0.1874527588813303, + "fcm_dpo/beta": 0.10289017111063004, + "fcm_dpo/delta": 0.027094706892967224, + "fcm_dpo/margin": 3.6252942085266113, + "fcm_dpo/q_t": 0.41391268372535706, + "grad_norm": 28.08007049560547, + "learning_rate": 4.89114813497619e-07, + "logits/chosen": 0.24639184772968292, + "logits/rejected": 0.18602319061756134, + "logps/chosen": -63.5056037902832, + "logps/ref_chosen": -57.116085052490234, + "logps/ref_rejected": -87.93074035644531, + "logps/rejected": -97.945556640625, + "loss": 1.1071, + "margin_dpo/margin_mean": 3.6252942085266113, + "margin_dpo/margin_std": 4.895936012268066, + "step": 124 + }, + { + "KL/chosen_KL_mean": -6.905740737915039, + "KL/mean": -8.502840042114258, + "KL/rejected_KL_mean": -10.099937438964844, + "KL/std": 4.83170223236084, + "epoch": 0.1889644746787604, + "fcm_dpo/beta": 0.10297717154026031, + "fcm_dpo/delta": 0.07129603624343872, + "fcm_dpo/margin": 3.1941990852355957, + "fcm_dpo/q_t": 0.42498207092285156, + "grad_norm": 29.70494842529297, + "learning_rate": 4.887255603610184e-07, + "logits/chosen": 0.2929185628890991, + "logits/rejected": 0.23179708421230316, + "logps/chosen": -72.61192321777344, + "logps/ref_chosen": -65.7061767578125, + "logps/ref_rejected": -91.72711944580078, + "logps/rejected": -101.82705688476562, + "loss": 1.1475, + "margin_dpo/margin_mean": 3.194199323654175, + "margin_dpo/margin_std": 5.006464958190918, + "step": 125 + }, + { + "KL/chosen_KL_mean": -6.534847259521484, + "KL/mean": -7.413846492767334, + "KL/rejected_KL_mean": -8.292850494384766, + "KL/std": 4.382688045501709, + "epoch": 0.19047619047619047, + "fcm_dpo/beta": 0.10306067764759064, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 1.7580021619796753, + "fcm_dpo/q_t": 0.4574311673641205, + "grad_norm": 29.526226043701172, + "learning_rate": 4.883296295573176e-07, + "logits/chosen": 0.11126873642206192, + "logits/rejected": 0.10555074363946915, + "logps/chosen": -74.7109375, + "logps/ref_chosen": -68.17608642578125, + "logps/ref_rejected": -65.1175537109375, + "logps/rejected": -73.410400390625, + "loss": 1.2687, + "margin_dpo/margin_mean": 1.7580022811889648, + "margin_dpo/margin_std": 4.669672012329102, + "step": 126 + }, + { + "KL/chosen_KL_mean": -6.717720031738281, + "KL/mean": -8.394515037536621, + "KL/rejected_KL_mean": -10.07131576538086, + "KL/std": 3.98907470703125, + "epoch": 0.19198790627362056, + "fcm_dpo/beta": 0.10309869796037674, + "fcm_dpo/delta": 0.05442150682210922, + "fcm_dpo/margin": 3.3535919189453125, + "fcm_dpo/q_t": 0.4188344180583954, + "grad_norm": 27.114885330200195, + "learning_rate": 4.87927032161552e-07, + "logits/chosen": 0.21206232905387878, + "logits/rejected": 0.181783527135849, + "logps/chosen": -68.59795379638672, + "logps/ref_chosen": -61.88023376464844, + "logps/ref_rejected": -68.46012878417969, + "logps/rejected": -78.53144073486328, + "loss": 1.1188, + "margin_dpo/margin_mean": 3.3535916805267334, + "margin_dpo/margin_std": 4.259637832641602, + "step": 127 + }, + { + "KL/chosen_KL_mean": -7.287038803100586, + "KL/mean": -9.122272491455078, + "KL/rejected_KL_mean": -10.957511901855469, + "KL/std": 5.400481700897217, + "epoch": 0.19349962207105065, + "fcm_dpo/beta": 0.10317796468734741, + "fcm_dpo/delta": 0.021360628306865692, + "fcm_dpo/margin": 3.6704673767089844, + "fcm_dpo/q_t": 0.4163186550140381, + "grad_norm": 28.42887306213379, + "learning_rate": 4.875177794352363e-07, + "logits/chosen": 0.25153741240501404, + "logits/rejected": 0.19308939576148987, + "logps/chosen": -73.99601745605469, + "logps/ref_chosen": -66.708984375, + "logps/ref_rejected": -94.97969055175781, + "logps/rejected": -105.93720245361328, + "loss": 1.1447, + "margin_dpo/margin_mean": 3.6704678535461426, + "margin_dpo/margin_std": 6.468405723571777, + "step": 128 + }, + { + "KL/chosen_KL_mean": -8.24397087097168, + "KL/mean": -9.266441345214844, + "KL/rejected_KL_mean": -10.28891372680664, + "KL/std": 4.9122772216796875, + "epoch": 0.19501133786848074, + "fcm_dpo/beta": 0.1032169982790947, + "fcm_dpo/delta": 0.0, + "fcm_dpo/margin": 2.0449447631835938, + "fcm_dpo/q_t": 0.4512721300125122, + "grad_norm": 35.81724548339844, + "learning_rate": 4.871018828260491e-07, + "logits/chosen": 0.23118102550506592, + "logits/rejected": 0.22530348598957062, + "logps/chosen": -73.58279418945312, + "logps/ref_chosen": -65.33882904052734, + "logps/ref_rejected": -68.06109619140625, + "logps/rejected": -78.35000610351562, + "loss": 1.2742, + "margin_dpo/margin_mean": 2.044945240020752, + "margin_dpo/margin_std": 5.920280456542969, + "step": 129 + }, + { + "KL/chosen_KL_mean": -7.786478042602539, + "KL/mean": -9.49015998840332, + "KL/rejected_KL_mean": -11.193840026855469, + "KL/std": 4.936343193054199, + "epoch": 0.1965230536659108, + "fcm_dpo/beta": 0.10324176400899887, + "fcm_dpo/delta": 0.04836462438106537, + "fcm_dpo/margin": 3.407360076904297, + "fcm_dpo/q_t": 0.4181935787200928, + "grad_norm": 31.07500648498535, + "learning_rate": 4.866793539675126e-07, + "logits/chosen": 0.19816309213638306, + "logits/rejected": 0.14642281830310822, + "logps/chosen": -66.44721984863281, + "logps/ref_chosen": -58.660743713378906, + "logps/ref_rejected": -79.24510192871094, + "logps/rejected": -90.4389419555664, + "loss": 1.1247, + "margin_dpo/margin_mean": 3.407360553741455, + "margin_dpo/margin_std": 4.771332740783691, + "step": 130 + }, + { + "KL/chosen_KL_mean": -7.415262222290039, + "KL/mean": -9.611775398254395, + "KL/rejected_KL_mean": -11.80828857421875, + "KL/std": 5.675230026245117, + "epoch": 0.1980347694633409, + "fcm_dpo/beta": 0.10321280360221863, + "fcm_dpo/delta": -0.05367963761091232, + "fcm_dpo/margin": 4.393032073974609, + "fcm_dpo/q_t": 0.3999623656272888, + "grad_norm": 26.43085479736328, + "learning_rate": 4.86250204678667e-07, + "logits/chosen": 0.2192288339138031, + "logits/rejected": 0.1527276486158371, + "logps/chosen": -59.9297981262207, + "logps/ref_chosen": -52.51453399658203, + "logps/ref_rejected": -85.18299865722656, + "logps/rejected": -96.99128723144531, + "loss": 1.0829, + "margin_dpo/margin_mean": 4.393032073974609, + "margin_dpo/margin_std": 6.335704326629639, + "step": 131 + }, + { + "KL/chosen_KL_mean": -8.307807922363281, + "KL/mean": -10.061725616455078, + "KL/rejected_KL_mean": -11.815641403198242, + "KL/std": 5.438101768493652, + "epoch": 0.19954648526077098, + "fcm_dpo/beta": 0.10326075553894043, + "fcm_dpo/delta": 0.037916913628578186, + "fcm_dpo/margin": 3.507833957672119, + "fcm_dpo/q_t": 0.4145070016384125, + "grad_norm": 28.417316436767578, + "learning_rate": 4.858144469637408e-07, + "logits/chosen": 0.28207463026046753, + "logits/rejected": 0.24967598915100098, + "logps/chosen": -73.99293518066406, + "logps/ref_chosen": -65.68513488769531, + "logps/ref_rejected": -69.54120635986328, + "logps/rejected": -81.35684967041016, + "loss": 1.1278, + "margin_dpo/margin_mean": 3.507833957672119, + "margin_dpo/margin_std": 5.2588043212890625, + "step": 132 + }, + { + "KL/chosen_KL_mean": -8.757793426513672, + "KL/mean": -10.143437385559082, + "KL/rejected_KL_mean": -11.52907943725586, + "KL/std": 5.314875602722168, + "epoch": 0.20105820105820105, + "fcm_dpo/beta": 0.1034662127494812, + "fcm_dpo/delta": 0.11358366161584854, + "fcm_dpo/margin": 2.7712912559509277, + "fcm_dpo/q_t": 0.434188574552536, + "grad_norm": 33.02809524536133, + "learning_rate": 4.853720930118138e-07, + "logits/chosen": 0.20251384377479553, + "logits/rejected": 0.19269287586212158, + "logps/chosen": -72.35591125488281, + "logps/ref_chosen": -63.598114013671875, + "logps/ref_rejected": -73.72798156738281, + "logps/rejected": -85.25706481933594, + "loss": 1.2067, + "margin_dpo/margin_mean": 2.7712912559509277, + "margin_dpo/margin_std": 5.831700325012207, + "step": 133 + }, + { + "KL/chosen_KL_mean": -8.120147705078125, + "KL/mean": -10.559319496154785, + "KL/rejected_KL_mean": -12.998485565185547, + "KL/std": 6.131566047668457, + "epoch": 0.20256991685563114, + "fcm_dpo/beta": 0.10336704552173615, + "fcm_dpo/delta": -0.10477904975414276, + "fcm_dpo/margin": 4.878334999084473, + "fcm_dpo/q_t": 0.3884613811969757, + "grad_norm": 26.100566864013672, + "learning_rate": 4.849231551964771e-07, + "logits/chosen": 0.318739116191864, + "logits/rejected": 0.2608543038368225, + "logps/chosen": -61.91471862792969, + "logps/ref_chosen": -53.79457092285156, + "logps/ref_rejected": -74.16741943359375, + "logps/rejected": -87.16590881347656, + "loss": 1.0377, + "margin_dpo/margin_mean": 4.878335475921631, + "margin_dpo/margin_std": 6.22467041015625, + "step": 134 + }, + { + "KL/chosen_KL_mean": -8.159568786621094, + "KL/mean": -9.734169006347656, + "KL/rejected_KL_mean": -11.30877685546875, + "KL/std": 5.708761692047119, + "epoch": 0.20408163265306123, + "fcm_dpo/beta": 0.10342980921268463, + "fcm_dpo/delta": 0.07450948655605316, + "fcm_dpo/margin": 3.1492090225219727, + "fcm_dpo/q_t": 0.4253659248352051, + "grad_norm": 26.205211639404297, + "learning_rate": 4.844676460754862e-07, + "logits/chosen": 0.2484242469072342, + "logits/rejected": 0.21588802337646484, + "logps/chosen": -57.60064697265625, + "logps/ref_chosen": -49.441078186035156, + "logps/ref_rejected": -65.96878051757812, + "logps/rejected": -77.27755737304688, + "loss": 1.1731, + "margin_dpo/margin_mean": 3.1492087841033936, + "margin_dpo/margin_std": 5.8807268142700195, + "step": 135 + }, + { + "KL/chosen_KL_mean": -10.12600326538086, + "KL/mean": -12.17684555053711, + "KL/rejected_KL_mean": -14.22768783569336, + "KL/std": 6.3857011795043945, + "epoch": 0.20559334845049132, + "fcm_dpo/beta": 0.10336458683013916, + "fcm_dpo/delta": -0.024140439927577972, + "fcm_dpo/margin": 4.101693153381348, + "fcm_dpo/q_t": 0.40940165519714355, + "grad_norm": 31.888391494750977, + "learning_rate": 4.840055783904106e-07, + "logits/chosen": 0.2441619485616684, + "logits/rejected": 0.1695641726255417, + "logps/chosen": -76.88526916503906, + "logps/ref_chosen": -66.75926208496094, + "logps/ref_rejected": -94.61787414550781, + "logps/rejected": -108.84556579589844, + "loss": 1.161, + "margin_dpo/margin_mean": 4.101694583892822, + "margin_dpo/margin_std": 7.955696105957031, + "step": 136 + }, + { + "KL/chosen_KL_mean": -9.42474365234375, + "KL/mean": -11.488272666931152, + "KL/rejected_KL_mean": -13.551799774169922, + "KL/std": 6.011587142944336, + "epoch": 0.20710506424792138, + "fcm_dpo/beta": 0.10340084880590439, + "fcm_dpo/delta": -0.026870137080550194, + "fcm_dpo/margin": 4.1270527839660645, + "fcm_dpo/q_t": 0.4059041738510132, + "grad_norm": 26.863759994506836, + "learning_rate": 4.835369650662767e-07, + "logits/chosen": 0.28351613879203796, + "logits/rejected": 0.25333186984062195, + "logps/chosen": -66.20854187011719, + "logps/ref_chosen": -56.78379821777344, + "logps/ref_rejected": -69.89952087402344, + "logps/rejected": -83.45132446289062, + "loss": 1.1033, + "margin_dpo/margin_mean": 4.127053737640381, + "margin_dpo/margin_std": 6.389741897583008, + "step": 137 + }, + { + "KL/chosen_KL_mean": -11.154241561889648, + "KL/mean": -12.932373046875, + "KL/rejected_KL_mean": -14.71051025390625, + "KL/std": 6.27994966506958, + "epoch": 0.20861678004535147, + "fcm_dpo/beta": 0.10339991748332977, + "fcm_dpo/delta": 0.03240242227911949, + "fcm_dpo/margin": 3.5562610626220703, + "fcm_dpo/q_t": 0.41714638471603394, + "grad_norm": 28.8984317779541, + "learning_rate": 4.830618192112065e-07, + "logits/chosen": 0.2769492268562317, + "logits/rejected": 0.23991017043590546, + "logps/chosen": -69.92025756835938, + "logps/ref_chosen": -58.766014099121094, + "logps/ref_rejected": -68.12371826171875, + "logps/rejected": -82.834228515625, + "loss": 1.1808, + "margin_dpo/margin_mean": 3.5562610626220703, + "margin_dpo/margin_std": 7.193914413452148, + "step": 138 + }, + { + "KL/chosen_KL_mean": -11.833564758300781, + "KL/mean": -13.922916412353516, + "KL/rejected_KL_mean": -16.01226806640625, + "KL/std": 6.134763717651367, + "epoch": 0.21012849584278157, + "fcm_dpo/beta": 0.1033773124217987, + "fcm_dpo/delta": -0.03212062269449234, + "fcm_dpo/margin": 4.178703308105469, + "fcm_dpo/q_t": 0.40225905179977417, + "grad_norm": 31.611188888549805, + "learning_rate": 4.825801541160509e-07, + "logits/chosen": 0.2360890507698059, + "logits/rejected": 0.20542971789836884, + "logps/chosen": -83.05915832519531, + "logps/ref_chosen": -71.2255859375, + "logps/ref_rejected": -82.1834716796875, + "logps/rejected": -98.19573974609375, + "loss": 1.0977, + "margin_dpo/margin_mean": 4.178703308105469, + "margin_dpo/margin_std": 6.276601791381836, + "step": 139 + }, + { + "KL/chosen_KL_mean": -10.748310089111328, + "KL/mean": -13.698071479797363, + "KL/rejected_KL_mean": -16.647830963134766, + "KL/std": 8.149213790893555, + "epoch": 0.21164021164021163, + "fcm_dpo/beta": 0.10300512611865997, + "fcm_dpo/delta": -0.20900794863700867, + "fcm_dpo/margin": 5.899516582489014, + "fcm_dpo/q_t": 0.37280064821243286, + "grad_norm": 32.60865020751953, + "learning_rate": 4.820919832540181e-07, + "logits/chosen": 0.24358946084976196, + "logits/rejected": 0.19462716579437256, + "logps/chosen": -74.02597045898438, + "logps/ref_chosen": -63.27766418457031, + "logps/ref_rejected": -83.30647277832031, + "logps/rejected": -99.95429992675781, + "loss": 1.062, + "margin_dpo/margin_mean": 5.8995161056518555, + "margin_dpo/margin_std": 9.14741325378418, + "step": 140 + }, + { + "KL/chosen_KL_mean": -11.99802017211914, + "KL/mean": -14.653268814086914, + "KL/rejected_KL_mean": -17.30852508544922, + "KL/std": 6.86690616607666, + "epoch": 0.21315192743764172, + "fcm_dpo/beta": 0.1026197224855423, + "fcm_dpo/delta": -0.14597390592098236, + "fcm_dpo/margin": 5.310500144958496, + "fcm_dpo/q_t": 0.387350857257843, + "grad_norm": 30.303489685058594, + "learning_rate": 4.815973202802966e-07, + "logits/chosen": 0.2936280369758606, + "logits/rejected": 0.24597673118114471, + "logps/chosen": -73.76478576660156, + "logps/ref_chosen": -61.76676940917969, + "logps/ref_rejected": -88.60601806640625, + "logps/rejected": -105.91455078125, + "loss": 1.0905, + "margin_dpo/margin_mean": 5.310500144958496, + "margin_dpo/margin_std": 8.6422119140625, + "step": 141 + }, + { + "KL/chosen_KL_mean": -11.66067123413086, + "KL/mean": -13.844879150390625, + "KL/rejected_KL_mean": -16.029090881347656, + "KL/std": 6.637794494628906, + "epoch": 0.2146636432350718, + "fcm_dpo/beta": 0.10252516716718674, + "fcm_dpo/delta": -0.04810721427202225, + "fcm_dpo/margin": 4.36842155456543, + "fcm_dpo/q_t": 0.4011050760746002, + "grad_norm": 28.417081832885742, + "learning_rate": 4.810961790316729e-07, + "logits/chosen": 0.3013097047805786, + "logits/rejected": 0.2751958966255188, + "logps/chosen": -76.9354476928711, + "logps/ref_chosen": -65.2747802734375, + "logps/ref_rejected": -81.1378173828125, + "logps/rejected": -97.16690826416016, + "loss": 1.0987, + "margin_dpo/margin_mean": 4.36842155456543, + "margin_dpo/margin_std": 6.821197509765625, + "step": 142 + }, + { + "KL/chosen_KL_mean": -12.478555679321289, + "KL/mean": -14.356403350830078, + "KL/rejected_KL_mean": -16.2342529296875, + "KL/std": 7.895826816558838, + "epoch": 0.2161753590325019, + "fcm_dpo/beta": 0.1025661900639534, + "fcm_dpo/delta": 0.014837449416518211, + "fcm_dpo/margin": 3.7556991577148438, + "fcm_dpo/q_t": 0.41392016410827637, + "grad_norm": 40.103614807128906, + "learning_rate": 4.805885735261454e-07, + "logits/chosen": 0.30660706758499146, + "logits/rejected": 0.2885975241661072, + "logps/chosen": -75.09638214111328, + "logps/ref_chosen": -62.617828369140625, + "logps/ref_rejected": -70.39239501953125, + "logps/rejected": -86.62664794921875, + "loss": 1.2387, + "margin_dpo/margin_mean": 3.7556991577148438, + "margin_dpo/margin_std": 9.277976036071777, + "step": 143 + }, + { + "KL/chosen_KL_mean": -12.654075622558594, + "KL/mean": -15.086525917053223, + "KL/rejected_KL_mean": -17.518966674804688, + "KL/std": 7.84127140045166, + "epoch": 0.21768707482993196, + "fcm_dpo/beta": 0.10235791653394699, + "fcm_dpo/delta": -0.09852702170610428, + "fcm_dpo/margin": 4.8648905754089355, + "fcm_dpo/q_t": 0.3993350863456726, + "grad_norm": 31.363941192626953, + "learning_rate": 4.800745179625307e-07, + "logits/chosen": 0.2831832468509674, + "logits/rejected": 0.2500700354576111, + "logps/chosen": -73.4567642211914, + "logps/ref_chosen": -60.80268859863281, + "logps/ref_rejected": -79.07284545898438, + "logps/rejected": -96.59181213378906, + "loss": 1.1358, + "margin_dpo/margin_mean": 4.864890098571777, + "margin_dpo/margin_std": 9.074819564819336, + "step": 144 + }, + { + "KL/chosen_KL_mean": -11.807121276855469, + "KL/mean": -14.392024993896484, + "KL/rejected_KL_mean": -16.9769287109375, + "KL/std": 8.012412071228027, + "epoch": 0.21919879062736206, + "fcm_dpo/beta": 0.1021338403224945, + "fcm_dpo/delta": -0.12872418761253357, + "fcm_dpo/margin": 5.169816970825195, + "fcm_dpo/q_t": 0.3886655271053314, + "grad_norm": 35.503334045410156, + "learning_rate": 4.795540267200686e-07, + "logits/chosen": 0.23382404446601868, + "logits/rejected": 0.24972864985466003, + "logps/chosen": -86.4185791015625, + "logps/ref_chosen": -74.61146545410156, + "logps/ref_rejected": -83.24461364746094, + "logps/rejected": -100.22154235839844, + "loss": 1.1347, + "margin_dpo/margin_mean": 5.169816970825195, + "margin_dpo/margin_std": 9.682452201843262, + "step": 145 + }, + { + "KL/chosen_KL_mean": -11.482681274414062, + "KL/mean": -14.073657035827637, + "KL/rejected_KL_mean": -16.664630889892578, + "KL/std": 7.7937164306640625, + "epoch": 0.22071050642479215, + "fcm_dpo/beta": 0.1019207239151001, + "fcm_dpo/delta": -0.12880808115005493, + "fcm_dpo/margin": 5.181951999664307, + "fcm_dpo/q_t": 0.3898138999938965, + "grad_norm": 30.710147857666016, + "learning_rate": 4.790271143580173e-07, + "logits/chosen": 0.22148147225379944, + "logits/rejected": 0.20749756693840027, + "logps/chosen": -69.32366943359375, + "logps/ref_chosen": -57.84098434448242, + "logps/ref_rejected": -67.47422790527344, + "logps/rejected": -84.13885498046875, + "loss": 1.097, + "margin_dpo/margin_mean": 5.181952476501465, + "margin_dpo/margin_std": 8.60902214050293, + "step": 146 + }, + { + "KL/chosen_KL_mean": -13.152883529663086, + "KL/mean": -15.26097297668457, + "KL/rejected_KL_mean": -17.369068145751953, + "KL/std": 8.376655578613281, + "epoch": 0.2222222222222222, + "fcm_dpo/beta": 0.1017833948135376, + "fcm_dpo/delta": -0.02926221303641796, + "fcm_dpo/margin": 4.216182708740234, + "fcm_dpo/q_t": 0.408529669046402, + "grad_norm": 37.494197845458984, + "learning_rate": 4.784937956152489e-07, + "logits/chosen": 0.23608390986919403, + "logits/rejected": 0.1923035979270935, + "logps/chosen": -79.96635437011719, + "logps/ref_chosen": -66.81346893310547, + "logps/ref_rejected": -81.1796875, + "logps/rejected": -98.54875946044922, + "loss": 1.1859, + "margin_dpo/margin_mean": 4.216182708740234, + "margin_dpo/margin_std": 8.9732084274292, + "step": 147 + }, + { + "KL/chosen_KL_mean": -11.913017272949219, + "KL/mean": -14.650152206420898, + "KL/rejected_KL_mean": -17.387283325195312, + "KL/std": 7.812748908996582, + "epoch": 0.2237339380196523, + "fcm_dpo/beta": 0.10157064348459244, + "fcm_dpo/delta": -0.15686734020709991, + "fcm_dpo/margin": 5.474270343780518, + "fcm_dpo/q_t": 0.382058322429657, + "grad_norm": 25.468215942382812, + "learning_rate": 4.779540854098347e-07, + "logits/chosen": 0.3902188837528229, + "logits/rejected": 0.3148149251937866, + "logps/chosen": -60.600772857666016, + "logps/ref_chosen": -48.6877555847168, + "logps/ref_rejected": -67.50503540039062, + "logps/rejected": -84.89231872558594, + "loss": 1.0791, + "margin_dpo/margin_mean": 5.474269866943359, + "margin_dpo/margin_std": 8.634872436523438, + "step": 148 + }, + { + "KL/chosen_KL_mean": -14.381063461303711, + "KL/mean": -17.366275787353516, + "KL/rejected_KL_mean": -20.351486206054688, + "KL/std": 8.17977523803711, + "epoch": 0.2252456538170824, + "fcm_dpo/beta": 0.10115090012550354, + "fcm_dpo/delta": -0.20512549579143524, + "fcm_dpo/margin": 5.970426082611084, + "fcm_dpo/q_t": 0.37026941776275635, + "grad_norm": 27.11240005493164, + "learning_rate": 4.774079988386296e-07, + "logits/chosen": 0.2454262524843216, + "logits/rejected": 0.19800639152526855, + "logps/chosen": -69.52484130859375, + "logps/ref_chosen": -55.143775939941406, + "logps/ref_rejected": -64.79888916015625, + "logps/rejected": -85.15037536621094, + "loss": 1.0062, + "margin_dpo/margin_mean": 5.970425605773926, + "margin_dpo/margin_std": 7.765328884124756, + "step": 149 + }, + { + "KL/chosen_KL_mean": -12.195348739624023, + "KL/mean": -16.144733428955078, + "KL/rejected_KL_mean": -20.0941162109375, + "KL/std": 9.010974884033203, + "epoch": 0.22675736961451248, + "fcm_dpo/beta": 0.10042740404605865, + "fcm_dpo/delta": -0.3964199423789978, + "fcm_dpo/margin": 7.898773670196533, + "fcm_dpo/q_t": 0.3422451615333557, + "grad_norm": 27.25328826904297, + "learning_rate": 4.768555511768486e-07, + "logits/chosen": 0.2615750730037689, + "logits/rejected": 0.21655285358428955, + "logps/chosen": -79.66609191894531, + "logps/ref_chosen": -67.47074890136719, + "logps/ref_rejected": -89.21170806884766, + "logps/rejected": -109.30582427978516, + "loss": 0.9294, + "margin_dpo/margin_mean": 7.898774147033691, + "margin_dpo/margin_std": 9.359230041503906, + "step": 150 + }, + { + "KL/chosen_KL_mean": -10.324472427368164, + "KL/mean": -14.791948318481445, + "KL/rejected_KL_mean": -19.259429931640625, + "KL/std": 9.622682571411133, + "epoch": 0.22826908541194255, + "fcm_dpo/beta": 0.0995093584060669, + "fcm_dpo/delta": -0.4934340715408325, + "fcm_dpo/margin": 8.934957504272461, + "fcm_dpo/q_t": 0.3238147497177124, + "grad_norm": 24.52779769897461, + "learning_rate": 4.762967578776406e-07, + "logits/chosen": 0.27360641956329346, + "logits/rejected": 0.2176380753517151, + "logps/chosen": -62.78401565551758, + "logps/ref_chosen": -52.45954132080078, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -98.32244110107422, + "loss": 0.8609, + "margin_dpo/margin_mean": 8.934957504272461, + "margin_dpo/margin_std": 9.657405853271484, + "step": 151 + }, + { + "KL/chosen_KL_mean": -13.204288482666016, + "KL/mean": -16.08668327331543, + "KL/rejected_KL_mean": -18.969074249267578, + "KL/std": 9.020400047302246, + "epoch": 0.22978080120937264, + "fcm_dpo/beta": 0.09907073527574539, + "fcm_dpo/delta": -0.17209270596504211, + "fcm_dpo/margin": 5.764786243438721, + "fcm_dpo/q_t": 0.3860609233379364, + "grad_norm": 29.81224250793457, + "learning_rate": 4.757316345716553e-07, + "logits/chosen": 0.3468804359436035, + "logits/rejected": 0.2916698157787323, + "logps/chosen": -69.75811767578125, + "logps/ref_chosen": -56.5538330078125, + "logps/ref_rejected": -76.55074310302734, + "logps/rejected": -95.51982116699219, + "loss": 1.1249, + "margin_dpo/margin_mean": 5.7647857666015625, + "margin_dpo/margin_std": 10.321913719177246, + "step": 152 + }, + { + "KL/chosen_KL_mean": -12.327163696289062, + "KL/mean": -15.675556182861328, + "KL/rejected_KL_mean": -19.023944854736328, + "KL/std": 9.246706008911133, + "epoch": 0.23129251700680273, + "fcm_dpo/beta": 0.0985221415758133, + "fcm_dpo/delta": -0.26150333881378174, + "fcm_dpo/margin": 6.696778297424316, + "fcm_dpo/q_t": 0.36234262585639954, + "grad_norm": 27.628662109375, + "learning_rate": 4.751601970666064e-07, + "logits/chosen": 0.2461574524641037, + "logits/rejected": 0.20957426726818085, + "logps/chosen": -80.33406066894531, + "logps/ref_chosen": -68.00689697265625, + "logps/ref_rejected": -74.83482360839844, + "logps/rejected": -93.8587646484375, + "loss": 0.9943, + "margin_dpo/margin_mean": 6.696778297424316, + "margin_dpo/margin_std": 8.798181533813477, + "step": 153 + }, + { + "KL/chosen_KL_mean": -13.346345901489258, + "KL/mean": -15.80352783203125, + "KL/rejected_KL_mean": -18.260711669921875, + "KL/std": 9.635418891906738, + "epoch": 0.2328042328042328, + "fcm_dpo/beta": 0.0982503592967987, + "fcm_dpo/delta": -0.08326356112957001, + "fcm_dpo/margin": 4.914369583129883, + "fcm_dpo/q_t": 0.3979361951351166, + "grad_norm": 30.43021583557129, + "learning_rate": 4.745824613468292e-07, + "logits/chosen": 0.31943947076797485, + "logits/rejected": 0.31679508090019226, + "logps/chosen": -72.56888580322266, + "logps/ref_chosen": -59.222537994384766, + "logps/ref_rejected": -64.19131469726562, + "logps/rejected": -82.4520263671875, + "loss": 1.1746, + "margin_dpo/margin_mean": 4.914368629455566, + "margin_dpo/margin_std": 9.985753059387207, + "step": 154 + }, + { + "KL/chosen_KL_mean": -14.17401123046875, + "KL/mean": -17.600597381591797, + "KL/rejected_KL_mean": -21.027179718017578, + "KL/std": 9.43724250793457, + "epoch": 0.23431594860166288, + "fcm_dpo/beta": 0.09776261448860168, + "fcm_dpo/delta": -0.2720787227153778, + "fcm_dpo/margin": 6.853167533874512, + "fcm_dpo/q_t": 0.37133949995040894, + "grad_norm": 30.40814208984375, + "learning_rate": 4.7399844357283393e-07, + "logits/chosen": 0.3092416524887085, + "logits/rejected": 0.28908270597457886, + "logps/chosen": -82.62870788574219, + "logps/ref_chosen": -68.45469665527344, + "logps/ref_rejected": -77.91763305664062, + "logps/rejected": -98.94481658935547, + "loss": 1.1079, + "margin_dpo/margin_mean": 6.853167533874512, + "margin_dpo/margin_std": 11.994901657104492, + "step": 155 + }, + { + "KL/chosen_KL_mean": -13.929441452026367, + "KL/mean": -18.027273178100586, + "KL/rejected_KL_mean": -22.125102996826172, + "KL/std": 10.610441207885742, + "epoch": 0.23582766439909297, + "fcm_dpo/beta": 0.09712938219308853, + "fcm_dpo/delta": -0.3991551399230957, + "fcm_dpo/margin": 8.195661544799805, + "fcm_dpo/q_t": 0.34260234236717224, + "grad_norm": 27.137714385986328, + "learning_rate": 4.7340816008085305e-07, + "logits/chosen": 0.27745482325553894, + "logits/rejected": 0.23165100812911987, + "logps/chosen": -81.19903564453125, + "logps/ref_chosen": -67.26959991455078, + "logps/ref_rejected": -86.95914459228516, + "logps/rejected": -109.08424377441406, + "loss": 0.9305, + "margin_dpo/margin_mean": 8.195661544799805, + "margin_dpo/margin_std": 9.939712524414062, + "step": 156 + }, + { + "KL/chosen_KL_mean": -12.333717346191406, + "KL/mean": -15.554601669311523, + "KL/rejected_KL_mean": -18.77548599243164, + "KL/std": 10.037962913513184, + "epoch": 0.23733938019652306, + "fcm_dpo/beta": 0.09645235538482666, + "fcm_dpo/delta": -0.22346851229667664, + "fcm_dpo/margin": 6.441762447357178, + "fcm_dpo/q_t": 0.37333279848098755, + "grad_norm": 28.82990837097168, + "learning_rate": 4.728116273823847e-07, + "logits/chosen": 0.2783495783805847, + "logits/rejected": 0.2599300742149353, + "logps/chosen": -67.1065902709961, + "logps/ref_chosen": -54.77287292480469, + "logps/ref_rejected": -63.87866973876953, + "logps/rejected": -82.65415954589844, + "loss": 1.0681, + "margin_dpo/margin_mean": 6.441762447357178, + "margin_dpo/margin_std": 9.796937942504883, + "step": 157 + }, + { + "KL/chosen_KL_mean": -13.680839538574219, + "KL/mean": -17.137577056884766, + "KL/rejected_KL_mean": -20.59430694580078, + "KL/std": 10.273170471191406, + "epoch": 0.23885109599395313, + "fcm_dpo/beta": 0.09611721336841583, + "fcm_dpo/delta": -0.2662354111671448, + "fcm_dpo/margin": 6.9134674072265625, + "fcm_dpo/q_t": 0.37016022205352783, + "grad_norm": 29.508346557617188, + "learning_rate": 4.7220886216373085e-07, + "logits/chosen": 0.3386998772621155, + "logits/rejected": 0.29939788579940796, + "logps/chosen": -78.60356140136719, + "logps/ref_chosen": -64.92271423339844, + "logps/ref_rejected": -82.23789978027344, + "logps/rejected": -102.83219909667969, + "loss": 1.0589, + "margin_dpo/margin_mean": 6.913466453552246, + "margin_dpo/margin_std": 10.930315971374512, + "step": 158 + }, + { + "KL/chosen_KL_mean": -13.760581970214844, + "KL/mean": -17.69038963317871, + "KL/rejected_KL_mean": -21.620193481445312, + "KL/std": 10.88300609588623, + "epoch": 0.24036281179138322, + "fcm_dpo/beta": 0.0955384373664856, + "fcm_dpo/delta": -0.3533981442451477, + "fcm_dpo/margin": 7.859614372253418, + "fcm_dpo/q_t": 0.35132092237472534, + "grad_norm": 31.337242126464844, + "learning_rate": 4.715998812855304e-07, + "logits/chosen": 0.3035956621170044, + "logits/rejected": 0.2644485831260681, + "logps/chosen": -70.80757141113281, + "logps/ref_chosen": -57.046993255615234, + "logps/ref_rejected": -73.32441711425781, + "logps/rejected": -94.94461059570312, + "loss": 1.0152, + "margin_dpo/margin_mean": 7.859613418579102, + "margin_dpo/margin_std": 11.44137954711914, + "step": 159 + }, + { + "KL/chosen_KL_mean": -15.426515579223633, + "KL/mean": -18.731592178344727, + "KL/rejected_KL_mean": -22.036670684814453, + "KL/std": 10.044414520263672, + "epoch": 0.2418745275888133, + "fcm_dpo/beta": 0.0949837788939476, + "fcm_dpo/delta": -0.22926974296569824, + "fcm_dpo/margin": 6.61015510559082, + "fcm_dpo/q_t": 0.38169610500335693, + "grad_norm": 28.10798454284668, + "learning_rate": 4.7098470178228755e-07, + "logits/chosen": 0.19165638089179993, + "logits/rejected": 0.1452564299106598, + "logps/chosen": -65.23342895507812, + "logps/ref_chosen": -49.806915283203125, + "logps/ref_rejected": -68.3370132446289, + "logps/rejected": -90.37368774414062, + "loss": 1.102, + "margin_dpo/margin_mean": 6.61015510559082, + "margin_dpo/margin_std": 11.653867721557617, + "step": 160 + }, + { + "KL/chosen_KL_mean": -15.095945358276367, + "KL/mean": -18.75388526916504, + "KL/rejected_KL_mean": -22.411827087402344, + "KL/std": 9.855433464050293, + "epoch": 0.24338624338624337, + "fcm_dpo/beta": 0.09446100890636444, + "fcm_dpo/delta": -0.29305022954940796, + "fcm_dpo/margin": 7.315883636474609, + "fcm_dpo/q_t": 0.3610680103302002, + "grad_norm": 26.705514907836914, + "learning_rate": 4.703633408618955e-07, + "logits/chosen": 0.3087644875049591, + "logits/rejected": 0.26988035440444946, + "logps/chosen": -67.596435546875, + "logps/ref_chosen": -52.50048828125, + "logps/ref_rejected": -66.04540252685547, + "logps/rejected": -88.45722961425781, + "loss": 1.0393, + "margin_dpo/margin_mean": 7.315883636474609, + "margin_dpo/margin_std": 11.02990436553955, + "step": 161 + }, + { + "KL/chosen_KL_mean": -16.03990936279297, + "KL/mean": -20.799524307250977, + "KL/rejected_KL_mean": -25.55914306640625, + "KL/std": 11.092716217041016, + "epoch": 0.24489795918367346, + "fcm_dpo/beta": 0.09360922127962112, + "fcm_dpo/delta": -0.4955092668533325, + "fcm_dpo/margin": 9.519235610961914, + "fcm_dpo/q_t": 0.3238350450992584, + "grad_norm": 28.485057830810547, + "learning_rate": 4.697358159051549e-07, + "logits/chosen": 0.35491442680358887, + "logits/rejected": 0.30406713485717773, + "logps/chosen": -85.50910186767578, + "logps/ref_chosen": -69.46919250488281, + "logps/ref_rejected": -92.00952911376953, + "logps/rejected": -117.56867218017578, + "loss": 0.9113, + "margin_dpo/margin_mean": 9.519235610961914, + "margin_dpo/margin_std": 11.425970077514648, + "step": 162 + }, + { + "KL/chosen_KL_mean": -13.804462432861328, + "KL/mean": -18.548547744750977, + "KL/rejected_KL_mean": -23.29262924194336, + "KL/std": 10.78026008605957, + "epoch": 0.24640967498110355, + "fcm_dpo/beta": 0.09284988790750504, + "fcm_dpo/delta": -0.4849868714809418, + "fcm_dpo/margin": 9.488163948059082, + "fcm_dpo/q_t": 0.3301030993461609, + "grad_norm": 25.85110855102539, + "learning_rate": 4.691021444652876e-07, + "logits/chosen": 0.28614428639411926, + "logits/rejected": 0.23749694228172302, + "logps/chosen": -64.41829681396484, + "logps/ref_chosen": -50.613834381103516, + "logps/ref_rejected": -74.62033081054688, + "logps/rejected": -97.91295623779297, + "loss": 0.9359, + "margin_dpo/margin_mean": 9.488163948059082, + "margin_dpo/margin_std": 11.308109283447266, + "step": 163 + }, + { + "KL/chosen_KL_mean": -14.94032096862793, + "KL/mean": -19.523334503173828, + "KL/rejected_KL_mean": -24.10634994506836, + "KL/std": 11.312816619873047, + "epoch": 0.24792139077853365, + "fcm_dpo/beta": 0.09193621575832367, + "fcm_dpo/delta": -0.44622060656547546, + "fcm_dpo/margin": 9.166027069091797, + "fcm_dpo/q_t": 0.3387375473976135, + "grad_norm": 25.200773239135742, + "learning_rate": 4.6846234426744624e-07, + "logits/chosen": 0.2593820095062256, + "logits/rejected": 0.19578561186790466, + "logps/chosen": -69.78843688964844, + "logps/ref_chosen": -54.848114013671875, + "logps/ref_rejected": -79.0630111694336, + "logps/rejected": -103.16935729980469, + "loss": 0.9758, + "margin_dpo/margin_mean": 9.166027069091797, + "margin_dpo/margin_std": 12.178564071655273, + "step": 164 + }, + { + "KL/chosen_KL_mean": -16.66638946533203, + "KL/mean": -20.76047134399414, + "KL/rejected_KL_mean": -24.85455322265625, + "KL/std": 11.303337097167969, + "epoch": 0.2494331065759637, + "fcm_dpo/beta": 0.09125003218650818, + "fcm_dpo/delta": -0.3496321141719818, + "fcm_dpo/margin": 8.188163757324219, + "fcm_dpo/q_t": 0.35198479890823364, + "grad_norm": 27.400609970092773, + "learning_rate": 4.678164332082175e-07, + "logits/chosen": 0.33185404539108276, + "logits/rejected": 0.2762041985988617, + "logps/chosen": -67.75559997558594, + "logps/ref_chosen": -51.089210510253906, + "logps/ref_rejected": -71.23370361328125, + "logps/rejected": -96.0882568359375, + "loss": 1.0416, + "margin_dpo/margin_mean": 8.188163757324219, + "margin_dpo/margin_std": 12.103445053100586, + "step": 165 + }, + { + "KL/chosen_KL_mean": -15.843442916870117, + "KL/mean": -19.023862838745117, + "KL/rejected_KL_mean": -22.204280853271484, + "KL/std": 11.114295959472656, + "epoch": 0.2509448223733938, + "fcm_dpo/beta": 0.09074525535106659, + "fcm_dpo/delta": -0.17831744253635406, + "fcm_dpo/margin": 6.36083984375, + "fcm_dpo/q_t": 0.38684898614883423, + "grad_norm": 30.845542907714844, + "learning_rate": 4.6716442935512214e-07, + "logits/chosen": 0.31017789244651794, + "logits/rejected": 0.21507307887077332, + "logps/chosen": -79.03425598144531, + "logps/ref_chosen": -63.19081115722656, + "logps/ref_rejected": -93.8402099609375, + "logps/rejected": -116.04449462890625, + "loss": 1.0892, + "margin_dpo/margin_mean": 6.36083984375, + "margin_dpo/margin_std": 10.658010482788086, + "step": 166 + }, + { + "KL/chosen_KL_mean": -13.438032150268555, + "KL/mean": -18.269426345825195, + "KL/rejected_KL_mean": -23.10082244873047, + "KL/std": 11.151466369628906, + "epoch": 0.25245653817082386, + "fcm_dpo/beta": 0.08997842669487, + "fcm_dpo/delta": -0.47406429052352905, + "fcm_dpo/margin": 9.662786483764648, + "fcm_dpo/q_t": 0.3294220566749573, + "grad_norm": 25.25689697265625, + "learning_rate": 4.6650635094610966e-07, + "logits/chosen": 0.22942158579826355, + "logits/rejected": 0.19497239589691162, + "logps/chosen": -72.3623046875, + "logps/ref_chosen": -58.92427062988281, + "logps/ref_rejected": -72.97377014160156, + "logps/rejected": -96.07459259033203, + "loss": 0.8893, + "margin_dpo/margin_mean": 9.662786483764648, + "margin_dpo/margin_std": 10.849441528320312, + "step": 167 + }, + { + "KL/chosen_KL_mean": -15.68846321105957, + "KL/mean": -19.450448989868164, + "KL/rejected_KL_mean": -23.212440490722656, + "KL/std": 11.777135848999023, + "epoch": 0.25396825396825395, + "fcm_dpo/beta": 0.08944668620824814, + "fcm_dpo/delta": -0.27491575479507446, + "fcm_dpo/margin": 7.523977756500244, + "fcm_dpo/q_t": 0.3696911334991455, + "grad_norm": 27.894454956054688, + "learning_rate": 4.6584221638904767e-07, + "logits/chosen": 0.2750155031681061, + "logits/rejected": 0.23802116513252258, + "logps/chosen": -81.33984375, + "logps/ref_chosen": -65.65138244628906, + "logps/ref_rejected": -79.71418762207031, + "logps/rejected": -102.92662811279297, + "loss": 1.0233, + "margin_dpo/margin_mean": 7.523977279663086, + "margin_dpo/margin_std": 10.949456214904785, + "step": 168 + }, + { + "KL/chosen_KL_mean": -12.854196548461914, + "KL/mean": -17.091754913330078, + "KL/rejected_KL_mean": -21.329307556152344, + "KL/std": 11.633987426757812, + "epoch": 0.25547996976568405, + "fcm_dpo/beta": 0.08887064456939697, + "fcm_dpo/delta": -0.355859637260437, + "fcm_dpo/margin": 8.47510814666748, + "fcm_dpo/q_t": 0.35965457558631897, + "grad_norm": 31.099884033203125, + "learning_rate": 4.651720442612075e-07, + "logits/chosen": 0.32425931096076965, + "logits/rejected": 0.2897077798843384, + "logps/chosen": -74.28005981445312, + "logps/ref_chosen": -61.425865173339844, + "logps/ref_rejected": -76.09590148925781, + "logps/rejected": -97.42520141601562, + "loss": 1.0537, + "margin_dpo/margin_mean": 8.475109100341797, + "margin_dpo/margin_std": 13.803487777709961, + "step": 169 + }, + { + "KL/chosen_KL_mean": -12.19082260131836, + "KL/mean": -16.442466735839844, + "KL/rejected_KL_mean": -20.694110870361328, + "KL/std": 11.121298789978027, + "epoch": 0.25699168556311414, + "fcm_dpo/beta": 0.0882720947265625, + "fcm_dpo/delta": -0.3531571626663208, + "fcm_dpo/margin": 8.503292083740234, + "fcm_dpo/q_t": 0.35489630699157715, + "grad_norm": 25.417301177978516, + "learning_rate": 4.6449585330874425e-07, + "logits/chosen": 0.2988799214363098, + "logits/rejected": 0.2961501479148865, + "logps/chosen": -68.84400939941406, + "logps/ref_chosen": -56.65319061279297, + "logps/ref_rejected": -63.45965576171875, + "logps/rejected": -84.15376281738281, + "loss": 1.0742, + "margin_dpo/margin_mean": 8.503292083740234, + "margin_dpo/margin_std": 13.600449562072754, + "step": 170 + }, + { + "KL/chosen_KL_mean": -13.645515441894531, + "KL/mean": -18.27761459350586, + "KL/rejected_KL_mean": -22.909713745117188, + "KL/std": 12.661592483520508, + "epoch": 0.2585034013605442, + "fcm_dpo/beta": 0.08742457628250122, + "fcm_dpo/delta": -0.41416728496551514, + "fcm_dpo/margin": 9.264201164245605, + "fcm_dpo/q_t": 0.34703579545021057, + "grad_norm": 27.912363052368164, + "learning_rate": 4.6381366244617224e-07, + "logits/chosen": 0.3496447801589966, + "logits/rejected": 0.29783889651298523, + "logps/chosen": -77.38027954101562, + "logps/ref_chosen": -63.73476028442383, + "logps/ref_rejected": -78.50328063964844, + "logps/rejected": -101.41299438476562, + "loss": 1.0564, + "margin_dpo/margin_mean": 9.264200210571289, + "margin_dpo/margin_std": 14.337952613830566, + "step": 171 + }, + { + "KL/chosen_KL_mean": -14.899620056152344, + "KL/mean": -19.200889587402344, + "KL/rejected_KL_mean": -23.502147674560547, + "KL/std": 12.590666770935059, + "epoch": 0.2600151171579743, + "fcm_dpo/beta": 0.08700421452522278, + "fcm_dpo/delta": -0.3509501516819, + "fcm_dpo/margin": 8.60252571105957, + "fcm_dpo/q_t": 0.3562316596508026, + "grad_norm": 27.247074127197266, + "learning_rate": 4.631254907558365e-07, + "logits/chosen": 0.3247148394584656, + "logits/rejected": 0.2659016251564026, + "logps/chosen": -67.10137939453125, + "logps/ref_chosen": -52.201759338378906, + "logps/ref_rejected": -82.85285949707031, + "logps/rejected": -106.35501098632812, + "loss": 1.0324, + "margin_dpo/margin_mean": 8.60252571105957, + "margin_dpo/margin_std": 12.878762245178223, + "step": 172 + }, + { + "KL/chosen_KL_mean": -13.891881942749023, + "KL/mean": -18.465965270996094, + "KL/rejected_KL_mean": -23.040050506591797, + "KL/std": 12.69556999206543, + "epoch": 0.2615268329554044, + "fcm_dpo/beta": 0.08610422909259796, + "fcm_dpo/delta": -0.39188864827156067, + "fcm_dpo/margin": 9.148174285888672, + "fcm_dpo/q_t": 0.3621525764465332, + "grad_norm": 28.12903594970703, + "learning_rate": 4.624313574873786e-07, + "logits/chosen": 0.34976938366889954, + "logits/rejected": 0.25832200050354004, + "logps/chosen": -69.32660675048828, + "logps/ref_chosen": -55.434722900390625, + "logps/ref_rejected": -77.81967163085938, + "logps/rejected": -100.85972595214844, + "loss": 1.1279, + "margin_dpo/margin_mean": 9.148174285888672, + "margin_dpo/margin_std": 15.838163375854492, + "step": 173 + }, + { + "KL/chosen_KL_mean": -15.286687850952148, + "KL/mean": -20.097599029541016, + "KL/rejected_KL_mean": -24.90850830078125, + "KL/std": 12.485860824584961, + "epoch": 0.26303854875283444, + "fcm_dpo/beta": 0.08550532907247543, + "fcm_dpo/delta": -0.42631781101226807, + "fcm_dpo/margin": 9.621818542480469, + "fcm_dpo/q_t": 0.343117892742157, + "grad_norm": 28.16497039794922, + "learning_rate": 4.61731282057198e-07, + "logits/chosen": 0.3049791753292084, + "logits/rejected": 0.2376536726951599, + "logps/chosen": -72.4586410522461, + "logps/ref_chosen": -57.17195129394531, + "logps/ref_rejected": -85.47578430175781, + "logps/rejected": -110.38429260253906, + "loss": 0.9914, + "margin_dpo/margin_mean": 9.621818542480469, + "margin_dpo/margin_std": 13.605627059936523, + "step": 174 + }, + { + "KL/chosen_KL_mean": -13.542539596557617, + "KL/mean": -18.85790252685547, + "KL/rejected_KL_mean": -24.173267364501953, + "KL/std": 12.634629249572754, + "epoch": 0.26455026455026454, + "fcm_dpo/beta": 0.08471880853176117, + "fcm_dpo/delta": -0.505054771900177, + "fcm_dpo/margin": 10.630727767944336, + "fcm_dpo/q_t": 0.34073692560195923, + "grad_norm": 26.806140899658203, + "learning_rate": 4.6102528404790965e-07, + "logits/chosen": 0.37399396300315857, + "logits/rejected": 0.34215161204338074, + "logps/chosen": -81.20816802978516, + "logps/ref_chosen": -67.6656265258789, + "logps/ref_rejected": -84.36766815185547, + "logps/rejected": -108.54093933105469, + "loss": 0.9786, + "margin_dpo/margin_mean": 10.630727767944336, + "margin_dpo/margin_std": 14.767560958862305, + "step": 175 + }, + { + "KL/chosen_KL_mean": -16.39118194580078, + "KL/mean": -19.81927490234375, + "KL/rejected_KL_mean": -23.24736785888672, + "KL/std": 13.32034683227539, + "epoch": 0.2660619803476946, + "fcm_dpo/beta": 0.08419062197208405, + "fcm_dpo/delta": -0.17851664125919342, + "fcm_dpo/margin": 6.8561906814575195, + "fcm_dpo/q_t": 0.3931282162666321, + "grad_norm": 35.735755920410156, + "learning_rate": 4.603133832077953e-07, + "logits/chosen": 0.2799184024333954, + "logits/rejected": 0.2532980740070343, + "logps/chosen": -94.24993896484375, + "logps/ref_chosen": -77.8587646484375, + "logps/ref_rejected": -81.08732604980469, + "logps/rejected": -104.3346939086914, + "loss": 1.2097, + "margin_dpo/margin_mean": 6.856190204620361, + "margin_dpo/margin_std": 14.80046272277832, + "step": 176 + }, + { + "KL/chosen_KL_mean": -12.543292999267578, + "KL/mean": -19.18335723876953, + "KL/rejected_KL_mean": -25.82342529296875, + "KL/std": 13.317384719848633, + "epoch": 0.2675736961451247, + "fcm_dpo/beta": 0.0832488089799881, + "fcm_dpo/delta": -0.713805079460144, + "fcm_dpo/margin": 13.280126571655273, + "fcm_dpo/q_t": 0.2935779094696045, + "grad_norm": 29.76000213623047, + "learning_rate": 4.5959559945025183e-07, + "logits/chosen": 0.41573256254196167, + "logits/rejected": 0.31886789202690125, + "logps/chosen": -67.76368713378906, + "logps/ref_chosen": -55.22039794921875, + "logps/ref_rejected": -92.54973602294922, + "logps/rejected": -118.37316131591797, + "loss": 0.7979, + "margin_dpo/margin_mean": 13.280126571655273, + "margin_dpo/margin_std": 13.62392807006836, + "step": 177 + }, + { + "KL/chosen_KL_mean": -14.576883316040039, + "KL/mean": -18.426517486572266, + "KL/rejected_KL_mean": -22.276153564453125, + "KL/std": 12.271912574768066, + "epoch": 0.2690854119425548, + "fcm_dpo/beta": 0.08259066939353943, + "fcm_dpo/delta": -0.23799234628677368, + "fcm_dpo/margin": 7.699267387390137, + "fcm_dpo/q_t": 0.3729979395866394, + "grad_norm": 30.068323135375977, + "learning_rate": 4.588719528532341e-07, + "logits/chosen": 0.30982112884521484, + "logits/rejected": 0.2584332227706909, + "logps/chosen": -75.38737487792969, + "logps/ref_chosen": -60.81049346923828, + "logps/ref_rejected": -81.12973022460938, + "logps/rejected": -103.4058837890625, + "loss": 1.0713, + "margin_dpo/margin_mean": 7.699267387390137, + "margin_dpo/margin_std": 12.144587516784668, + "step": 178 + }, + { + "KL/chosen_KL_mean": -15.081012725830078, + "KL/mean": -19.794164657592773, + "KL/rejected_KL_mean": -24.507320404052734, + "KL/std": 12.630709648132324, + "epoch": 0.2705971277399849, + "fcm_dpo/beta": 0.08212892711162567, + "fcm_dpo/delta": -0.37708958983421326, + "fcm_dpo/margin": 9.426307678222656, + "fcm_dpo/q_t": 0.3534626364707947, + "grad_norm": 30.549224853515625, + "learning_rate": 4.581424636586928e-07, + "logits/chosen": 0.3358982801437378, + "logits/rejected": 0.3192867040634155, + "logps/chosen": -80.75273132324219, + "logps/ref_chosen": -65.67171478271484, + "logps/ref_rejected": -75.32586669921875, + "logps/rejected": -99.83319091796875, + "loss": 1.0447, + "margin_dpo/margin_mean": 9.426307678222656, + "margin_dpo/margin_std": 14.78122329711914, + "step": 179 + }, + { + "KL/chosen_KL_mean": -11.573627471923828, + "KL/mean": -15.449760437011719, + "KL/rejected_KL_mean": -19.32590103149414, + "KL/std": 12.113849639892578, + "epoch": 0.272108843537415, + "fcm_dpo/beta": 0.08171549439430237, + "fcm_dpo/delta": -0.23490986227989197, + "fcm_dpo/margin": 7.7522759437561035, + "fcm_dpo/q_t": 0.3795109987258911, + "grad_norm": 26.659095764160156, + "learning_rate": 4.5740715227200897e-07, + "logits/chosen": 0.1699858158826828, + "logits/rejected": 0.15040446817874908, + "logps/chosen": -68.25643920898438, + "logps/ref_chosen": -56.68280792236328, + "logps/ref_rejected": -64.94414520263672, + "logps/rejected": -84.27005004882812, + "loss": 1.1208, + "margin_dpo/margin_mean": 7.7522759437561035, + "margin_dpo/margin_std": 14.111481666564941, + "step": 180 + }, + { + "KL/chosen_KL_mean": -10.128362655639648, + "KL/mean": -15.723772048950195, + "KL/rejected_KL_mean": -21.319171905517578, + "KL/std": 12.99374771118164, + "epoch": 0.273620559334845, + "fcm_dpo/beta": 0.08095923066139221, + "fcm_dpo/delta": -0.5106788277626038, + "fcm_dpo/margin": 11.190811157226562, + "fcm_dpo/q_t": 0.32457318902015686, + "grad_norm": 25.103832244873047, + "learning_rate": 4.566660392614228e-07, + "logits/chosen": 0.3399677872657776, + "logits/rejected": 0.29876863956451416, + "logps/chosen": -70.90441131591797, + "logps/ref_chosen": -60.77604675292969, + "logps/ref_rejected": -83.98361206054688, + "logps/rejected": -105.30278778076172, + "loss": 0.8678, + "margin_dpo/margin_mean": 11.190810203552246, + "margin_dpo/margin_std": 12.438660621643066, + "step": 181 + }, + { + "KL/chosen_KL_mean": -13.326507568359375, + "KL/mean": -19.073650360107422, + "KL/rejected_KL_mean": -24.82079315185547, + "KL/std": 13.47926139831543, + "epoch": 0.2751322751322751, + "fcm_dpo/beta": 0.08012458682060242, + "fcm_dpo/delta": -0.5258299112319946, + "fcm_dpo/margin": 11.494285583496094, + "fcm_dpo/q_t": 0.3308119475841522, + "grad_norm": 27.138328552246094, + "learning_rate": 4.5591914535745817e-07, + "logits/chosen": 0.31853342056274414, + "logits/rejected": 0.23440571129322052, + "logps/chosen": -73.58029174804688, + "logps/ref_chosen": -60.2537841796875, + "logps/ref_rejected": -89.7706298828125, + "logps/rejected": -114.59142303466797, + "loss": 0.9654, + "margin_dpo/margin_mean": 11.494285583496094, + "margin_dpo/margin_std": 15.407928466796875, + "step": 182 + }, + { + "KL/chosen_KL_mean": -16.266420364379883, + "KL/mean": -18.65563201904297, + "KL/rejected_KL_mean": -21.044841766357422, + "KL/std": 12.814361572265625, + "epoch": 0.2766439909297052, + "fcm_dpo/beta": 0.07990828156471252, + "fcm_dpo/delta": 0.01821490004658699, + "fcm_dpo/margin": 4.778428077697754, + "fcm_dpo/q_t": 0.42014068365097046, + "grad_norm": 29.39368438720703, + "learning_rate": 4.551664914523433e-07, + "logits/chosen": 0.2914312481880188, + "logits/rejected": 0.2695249319076538, + "logps/chosen": -78.02783966064453, + "logps/ref_chosen": -61.76142120361328, + "logps/ref_rejected": -72.54627990722656, + "logps/rejected": -93.59112548828125, + "loss": 1.2678, + "margin_dpo/margin_mean": 4.7784271240234375, + "margin_dpo/margin_std": 12.737541198730469, + "step": 183 + }, + { + "KL/chosen_KL_mean": -10.225717544555664, + "KL/mean": -14.392915725708008, + "KL/rejected_KL_mean": -18.56011199951172, + "KL/std": 10.851048469543457, + "epoch": 0.2781557067271353, + "fcm_dpo/beta": 0.07955743372440338, + "fcm_dpo/delta": -0.26520317792892456, + "fcm_dpo/margin": 8.334392547607422, + "fcm_dpo/q_t": 0.3680853843688965, + "grad_norm": 23.290220260620117, + "learning_rate": 4.544080985994258e-07, + "logits/chosen": 0.39962106943130493, + "logits/rejected": 0.33347806334495544, + "logps/chosen": -57.066436767578125, + "logps/ref_chosen": -46.840721130371094, + "logps/ref_rejected": -69.3609390258789, + "logps/rejected": -87.92105102539062, + "loss": 0.9986, + "margin_dpo/margin_mean": 8.334392547607422, + "margin_dpo/margin_std": 11.119543075561523, + "step": 184 + }, + { + "KL/chosen_KL_mean": -12.676387786865234, + "KL/mean": -17.241352081298828, + "KL/rejected_KL_mean": -21.80630874633789, + "KL/std": 12.782798767089844, + "epoch": 0.2796674225245654, + "fcm_dpo/beta": 0.07906308770179749, + "fcm_dpo/delta": -0.3246428966522217, + "fcm_dpo/margin": 9.129922866821289, + "fcm_dpo/q_t": 0.36516422033309937, + "grad_norm": 24.793489456176758, + "learning_rate": 4.5364398801258394e-07, + "logits/chosen": 0.33074915409088135, + "logits/rejected": 0.28292813897132874, + "logps/chosen": -64.99752807617188, + "logps/ref_chosen": -52.32114028930664, + "logps/ref_rejected": -68.3885726928711, + "logps/rejected": -90.19488525390625, + "loss": 1.1153, + "margin_dpo/margin_mean": 9.129922866821289, + "margin_dpo/margin_std": 15.799885749816895, + "step": 185 + }, + { + "KL/chosen_KL_mean": -11.109125137329102, + "KL/mean": -16.165748596191406, + "KL/rejected_KL_mean": -21.222373962402344, + "KL/std": 12.628129005432129, + "epoch": 0.2811791383219955, + "fcm_dpo/beta": 0.07852091640233994, + "fcm_dpo/delta": -0.39734992384910583, + "fcm_dpo/margin": 10.113248825073242, + "fcm_dpo/q_t": 0.34954291582107544, + "grad_norm": 31.158998489379883, + "learning_rate": 4.5287418106563354e-07, + "logits/chosen": 0.27662771940231323, + "logits/rejected": 0.2326260805130005, + "logps/chosen": -78.52925109863281, + "logps/ref_chosen": -67.42012786865234, + "logps/ref_rejected": -82.50968933105469, + "logps/rejected": -103.73206329345703, + "loss": 1.0252, + "margin_dpo/margin_mean": 10.113249778747559, + "margin_dpo/margin_std": 15.291001319885254, + "step": 186 + }, + { + "KL/chosen_KL_mean": -13.50363540649414, + "KL/mean": -17.96316909790039, + "KL/rejected_KL_mean": -22.42270278930664, + "KL/std": 13.12786865234375, + "epoch": 0.28269085411942557, + "fcm_dpo/beta": 0.0780055820941925, + "fcm_dpo/delta": -0.2979331612586975, + "fcm_dpo/margin": 8.919073104858398, + "fcm_dpo/q_t": 0.37085196375846863, + "grad_norm": 31.343835830688477, + "learning_rate": 4.520986992917297e-07, + "logits/chosen": 0.2847750186920166, + "logits/rejected": 0.23050551116466522, + "logps/chosen": -89.02912902832031, + "logps/ref_chosen": -75.52549743652344, + "logps/ref_rejected": -94.76289367675781, + "logps/rejected": -117.18559265136719, + "loss": 1.0926, + "margin_dpo/margin_mean": 8.919073104858398, + "margin_dpo/margin_std": 15.208412170410156, + "step": 187 + }, + { + "KL/chosen_KL_mean": -11.557426452636719, + "KL/mean": -16.42307472229004, + "KL/rejected_KL_mean": -21.28872299194336, + "KL/std": 12.959375381469727, + "epoch": 0.2842025699168556, + "fcm_dpo/beta": 0.07751020789146423, + "fcm_dpo/delta": -0.35692399740219116, + "fcm_dpo/margin": 9.731300354003906, + "fcm_dpo/q_t": 0.35627931356430054, + "grad_norm": 26.84238624572754, + "learning_rate": 4.5131756438276466e-07, + "logits/chosen": 0.30408740043640137, + "logits/rejected": 0.2604471445083618, + "logps/chosen": -83.08075714111328, + "logps/ref_chosen": -71.52333068847656, + "logps/ref_rejected": -78.29949951171875, + "logps/rejected": -99.58822631835938, + "loss": 1.0102, + "margin_dpo/margin_mean": 9.731300354003906, + "margin_dpo/margin_std": 14.259878158569336, + "step": 188 + }, + { + "KL/chosen_KL_mean": -11.125009536743164, + "KL/mean": -15.699823379516602, + "KL/rejected_KL_mean": -20.274639129638672, + "KL/std": 12.906343460083008, + "epoch": 0.2857142857142857, + "fcm_dpo/beta": 0.07690228521823883, + "fcm_dpo/delta": -0.3065638840198517, + "fcm_dpo/margin": 9.149633407592773, + "fcm_dpo/q_t": 0.3611648678779602, + "grad_norm": 27.982040405273438, + "learning_rate": 4.5053079818876096e-07, + "logits/chosen": 0.3254922032356262, + "logits/rejected": 0.3387814462184906, + "logps/chosen": -83.30127716064453, + "logps/ref_chosen": -72.17626953125, + "logps/ref_rejected": -75.26313781738281, + "logps/rejected": -95.53777313232422, + "loss": 1.0317, + "margin_dpo/margin_mean": 9.14963436126709, + "margin_dpo/margin_std": 13.340032577514648, + "step": 189 + }, + { + "KL/chosen_KL_mean": -10.58774185180664, + "KL/mean": -16.31813621520996, + "KL/rejected_KL_mean": -22.04853057861328, + "KL/std": 12.940621376037598, + "epoch": 0.2872260015117158, + "fcm_dpo/beta": 0.07633734494447708, + "fcm_dpo/delta": -0.47908633947372437, + "fcm_dpo/margin": 11.460789680480957, + "fcm_dpo/q_t": 0.3331252634525299, + "grad_norm": 30.75900650024414, + "learning_rate": 4.4973842271726024e-07, + "logits/chosen": 0.38086292147636414, + "logits/rejected": 0.2351219654083252, + "logps/chosen": -65.2120132446289, + "logps/ref_chosen": -54.624271392822266, + "logps/ref_rejected": -101.47068786621094, + "logps/rejected": -123.51921844482422, + "loss": 0.9067, + "margin_dpo/margin_mean": 11.46078872680664, + "margin_dpo/margin_std": 13.832509994506836, + "step": 190 + }, + { + "KL/chosen_KL_mean": -14.451740264892578, + "KL/mean": -19.122814178466797, + "KL/rejected_KL_mean": -23.79388427734375, + "KL/std": 13.62421989440918, + "epoch": 0.2887377173091459, + "fcm_dpo/beta": 0.07579559087753296, + "fcm_dpo/delta": -0.3103182911872864, + "fcm_dpo/margin": 9.342151641845703, + "fcm_dpo/q_t": 0.3602200746536255, + "grad_norm": 31.50908660888672, + "learning_rate": 4.48940460132708e-07, + "logits/chosen": 0.34070515632629395, + "logits/rejected": 0.3121827244758606, + "logps/chosen": -87.3842544555664, + "logps/ref_chosen": -72.93251037597656, + "logps/ref_rejected": -89.95103454589844, + "logps/rejected": -113.74491882324219, + "loss": 1.0607, + "margin_dpo/margin_mean": 9.342151641845703, + "margin_dpo/margin_std": 14.921095848083496, + "step": 191 + }, + { + "KL/chosen_KL_mean": -15.045526504516602, + "KL/mean": -17.96548843383789, + "KL/rejected_KL_mean": -20.885452270507812, + "KL/std": 13.691113471984863, + "epoch": 0.29024943310657597, + "fcm_dpo/beta": 0.07566338777542114, + "fcm_dpo/delta": -0.04206560552120209, + "fcm_dpo/margin": 5.839931488037109, + "fcm_dpo/q_t": 0.4071265757083893, + "grad_norm": 23.48601531982422, + "learning_rate": 4.481369327558329e-07, + "logits/chosen": 0.3627796173095703, + "logits/rejected": 0.33564233779907227, + "logps/chosen": -69.04664611816406, + "logps/ref_chosen": -54.001121520996094, + "logps/ref_rejected": -63.531551361083984, + "logps/rejected": -84.41700744628906, + "loss": 1.1695, + "margin_dpo/margin_mean": 5.839931488037109, + "margin_dpo/margin_std": 11.908516883850098, + "step": 192 + }, + { + "KL/chosen_KL_mean": -12.220426559448242, + "KL/mean": -17.491397857666016, + "KL/rejected_KL_mean": -22.76236343383789, + "KL/std": 12.884895324707031, + "epoch": 0.29176114890400606, + "fcm_dpo/beta": 0.07511965930461884, + "fcm_dpo/delta": -0.3954230844974518, + "fcm_dpo/margin": 10.541938781738281, + "fcm_dpo/q_t": 0.34566789865493774, + "grad_norm": 23.28472137451172, + "learning_rate": 4.47327863063023e-07, + "logits/chosen": 0.2976009249687195, + "logits/rejected": 0.27398666739463806, + "logps/chosen": -68.9697036743164, + "logps/ref_chosen": -56.74927520751953, + "logps/ref_rejected": -58.80629348754883, + "logps/rejected": -81.56865692138672, + "loss": 0.9452, + "margin_dpo/margin_mean": 10.541938781738281, + "margin_dpo/margin_std": 13.353843688964844, + "step": 193 + }, + { + "KL/chosen_KL_mean": -12.281147003173828, + "KL/mean": -16.789499282836914, + "KL/rejected_KL_mean": -21.297855377197266, + "KL/std": 12.686100006103516, + "epoch": 0.29327286470143615, + "fcm_dpo/beta": 0.07462107390165329, + "fcm_dpo/delta": -0.27535536885261536, + "fcm_dpo/margin": 9.016707420349121, + "fcm_dpo/q_t": 0.3693960905075073, + "grad_norm": 24.402481079101562, + "learning_rate": 4.4651327368569684e-07, + "logits/chosen": 0.3564680218696594, + "logits/rejected": 0.3261492848396301, + "logps/chosen": -68.93058776855469, + "logps/ref_chosen": -56.64944076538086, + "logps/ref_rejected": -69.98954772949219, + "logps/rejected": -91.28739929199219, + "loss": 1.0949, + "margin_dpo/margin_mean": 9.016707420349121, + "margin_dpo/margin_std": 14.944689750671387, + "step": 194 + }, + { + "KL/chosen_KL_mean": -15.373340606689453, + "KL/mean": -20.619035720825195, + "KL/rejected_KL_mean": -25.864734649658203, + "KL/std": 14.731451988220215, + "epoch": 0.2947845804988662, + "fcm_dpo/beta": 0.07414625585079193, + "fcm_dpo/delta": -0.38113653659820557, + "fcm_dpo/margin": 10.491392135620117, + "fcm_dpo/q_t": 0.3485524356365204, + "grad_norm": 27.90333366394043, + "learning_rate": 4.4569318740967043e-07, + "logits/chosen": 0.2557808458805084, + "logits/rejected": 0.25931698083877563, + "logps/chosen": -85.78311920166016, + "logps/ref_chosen": -70.40977478027344, + "logps/ref_rejected": -74.39448547363281, + "logps/rejected": -100.25921630859375, + "loss": 0.9879, + "margin_dpo/margin_mean": 10.491392135620117, + "margin_dpo/margin_std": 14.370185852050781, + "step": 195 + }, + { + "KL/chosen_KL_mean": -14.235525131225586, + "KL/mean": -18.705707550048828, + "KL/rejected_KL_mean": -23.175891876220703, + "KL/std": 14.081775665283203, + "epoch": 0.2962962962962963, + "fcm_dpo/beta": 0.07381300628185272, + "fcm_dpo/delta": -0.2615591585636139, + "fcm_dpo/margin": 8.940363883972168, + "fcm_dpo/q_t": 0.36146679520606995, + "grad_norm": 23.71242904663086, + "learning_rate": 4.448676271745197e-07, + "logits/chosen": 0.36908388137817383, + "logits/rejected": 0.32557791471481323, + "logps/chosen": -73.46310424804688, + "logps/ref_chosen": -59.227577209472656, + "logps/ref_rejected": -83.54757690429688, + "logps/rejected": -106.72346496582031, + "loss": 1.0202, + "margin_dpo/margin_mean": 8.940362930297852, + "margin_dpo/margin_std": 12.490043640136719, + "step": 196 + }, + { + "KL/chosen_KL_mean": -12.557924270629883, + "KL/mean": -17.917882919311523, + "KL/rejected_KL_mean": -23.27783966064453, + "KL/std": 13.854819297790527, + "epoch": 0.29780801209372637, + "fcm_dpo/beta": 0.07331664860248566, + "fcm_dpo/delta": -0.38882410526275635, + "fcm_dpo/margin": 10.719913482666016, + "fcm_dpo/q_t": 0.3505423665046692, + "grad_norm": 25.433277130126953, + "learning_rate": 4.440366160729392e-07, + "logits/chosen": 0.45952826738357544, + "logits/rejected": 0.4059959053993225, + "logps/chosen": -64.08705139160156, + "logps/ref_chosen": -51.52912902832031, + "logps/ref_rejected": -73.70631408691406, + "logps/rejected": -96.98414611816406, + "loss": 1.0669, + "margin_dpo/margin_mean": 10.719914436340332, + "margin_dpo/margin_std": 16.79769515991211, + "step": 197 + }, + { + "KL/chosen_KL_mean": -13.416961669921875, + "KL/mean": -18.900814056396484, + "KL/rejected_KL_mean": -24.384666442871094, + "KL/std": 13.024833679199219, + "epoch": 0.29931972789115646, + "fcm_dpo/beta": 0.07271362841129303, + "fcm_dpo/delta": -0.4005015790462494, + "fcm_dpo/margin": 10.967698097229004, + "fcm_dpo/q_t": 0.3381541669368744, + "grad_norm": 23.88762855529785, + "learning_rate": 4.432001773500957e-07, + "logits/chosen": 0.4004214406013489, + "logits/rejected": 0.35833674669265747, + "logps/chosen": -73.19964599609375, + "logps/ref_chosen": -59.78268051147461, + "logps/ref_rejected": -72.24533081054688, + "logps/rejected": -96.62998962402344, + "loss": 0.9278, + "margin_dpo/margin_mean": 10.96769905090332, + "margin_dpo/margin_std": 12.954378128051758, + "step": 198 + }, + { + "KL/chosen_KL_mean": -15.869409561157227, + "KL/mean": -20.172086715698242, + "KL/rejected_KL_mean": -24.47476577758789, + "KL/std": 13.327393531799316, + "epoch": 0.30083144368858655, + "fcm_dpo/beta": 0.07235260307788849, + "fcm_dpo/delta": -0.22404484450817108, + "fcm_dpo/margin": 8.605356216430664, + "fcm_dpo/q_t": 0.37417128682136536, + "grad_norm": 25.43196678161621, + "learning_rate": 4.4235833440297856e-07, + "logits/chosen": 0.34006598591804504, + "logits/rejected": 0.2518029808998108, + "logps/chosen": -72.25617980957031, + "logps/ref_chosen": -56.38677215576172, + "logps/ref_rejected": -74.56779479980469, + "logps/rejected": -99.04256439208984, + "loss": 1.125, + "margin_dpo/margin_mean": 8.605356216430664, + "margin_dpo/margin_std": 15.0706787109375, + "step": 199 + }, + { + "KL/chosen_KL_mean": -12.22726058959961, + "KL/mean": -18.615333557128906, + "KL/rejected_KL_mean": -25.003406524658203, + "KL/std": 14.295297622680664, + "epoch": 0.30234315948601664, + "fcm_dpo/beta": 0.07157911360263824, + "fcm_dpo/delta": -0.5196402072906494, + "fcm_dpo/margin": 12.776147842407227, + "fcm_dpo/q_t": 0.33188551664352417, + "grad_norm": 23.506271362304688, + "learning_rate": 4.415111107797445e-07, + "logits/chosen": 0.39532995223999023, + "logits/rejected": 0.321277379989624, + "logps/chosen": -70.05158996582031, + "logps/ref_chosen": -57.82432556152344, + "logps/ref_rejected": -89.28246307373047, + "logps/rejected": -114.28587341308594, + "loss": 0.954, + "margin_dpo/margin_mean": 12.776147842407227, + "margin_dpo/margin_std": 17.001556396484375, + "step": 200 + }, + { + "KL/chosen_KL_mean": -17.024478912353516, + "KL/mean": -21.887775421142578, + "KL/rejected_KL_mean": -26.751075744628906, + "KL/std": 14.819794654846191, + "epoch": 0.30385487528344673, + "fcm_dpo/beta": 0.0710674524307251, + "fcm_dpo/delta": -0.2936139702796936, + "fcm_dpo/margin": 9.726594924926758, + "fcm_dpo/q_t": 0.36819079518318176, + "grad_norm": 27.878252029418945, + "learning_rate": 4.4065853017905953e-07, + "logits/chosen": 0.3978556990623474, + "logits/rejected": 0.3516915440559387, + "logps/chosen": -76.02423858642578, + "logps/ref_chosen": -58.999759674072266, + "logps/ref_rejected": -84.67575073242188, + "logps/rejected": -111.42681884765625, + "loss": 1.0642, + "margin_dpo/margin_mean": 9.726594924926758, + "margin_dpo/margin_std": 15.60273551940918, + "step": 201 + }, + { + "KL/chosen_KL_mean": -14.108480453491211, + "KL/mean": -19.51925277709961, + "KL/rejected_KL_mean": -24.930023193359375, + "KL/std": 13.84997844696045, + "epoch": 0.30536659108087677, + "fcm_dpo/beta": 0.07067304849624634, + "fcm_dpo/delta": -0.3674495220184326, + "fcm_dpo/margin": 10.821544647216797, + "fcm_dpo/q_t": 0.3458291292190552, + "grad_norm": 25.05854034423828, + "learning_rate": 4.3980061644943575e-07, + "logits/chosen": 0.3353304862976074, + "logits/rejected": 0.2603084444999695, + "logps/chosen": -61.769126892089844, + "logps/ref_chosen": -47.660648345947266, + "logps/ref_rejected": -73.63249969482422, + "logps/rejected": -98.5625228881836, + "loss": 0.9989, + "margin_dpo/margin_mean": 10.821544647216797, + "margin_dpo/margin_std": 15.139564514160156, + "step": 202 + }, + { + "KL/chosen_KL_mean": -16.547443389892578, + "KL/mean": -21.438800811767578, + "KL/rejected_KL_mean": -26.330158233642578, + "KL/std": 14.137290954589844, + "epoch": 0.30687830687830686, + "fcm_dpo/beta": 0.07021065056324005, + "fcm_dpo/delta": -0.288774698972702, + "fcm_dpo/margin": 9.782716751098633, + "fcm_dpo/q_t": 0.36650407314300537, + "grad_norm": 27.474205017089844, + "learning_rate": 4.3893739358856455e-07, + "logits/chosen": 0.41657450795173645, + "logits/rejected": 0.33698996901512146, + "logps/chosen": -78.87297821044922, + "logps/ref_chosen": -62.32553482055664, + "logps/ref_rejected": -99.37226104736328, + "logps/rejected": -125.70242309570312, + "loss": 1.0468, + "margin_dpo/margin_mean": 9.782716751098633, + "margin_dpo/margin_std": 15.285531997680664, + "step": 203 + }, + { + "KL/chosen_KL_mean": -15.409299850463867, + "KL/mean": -20.86515235900879, + "KL/rejected_KL_mean": -26.321002960205078, + "KL/std": 14.736394882202148, + "epoch": 0.30839002267573695, + "fcm_dpo/beta": 0.0695991963148117, + "fcm_dpo/delta": -0.3632936179637909, + "fcm_dpo/margin": 10.911706924438477, + "fcm_dpo/q_t": 0.3535752594470978, + "grad_norm": 23.503171920776367, + "learning_rate": 4.380688857426449e-07, + "logits/chosen": 0.3589133620262146, + "logits/rejected": 0.28745904564857483, + "logps/chosen": -66.03861999511719, + "logps/ref_chosen": -50.62931823730469, + "logps/ref_rejected": -66.60475158691406, + "logps/rejected": -92.92575073242188, + "loss": 0.9949, + "margin_dpo/margin_mean": 10.91170597076416, + "margin_dpo/margin_std": 14.736675262451172, + "step": 204 + }, + { + "KL/chosen_KL_mean": -17.15616226196289, + "KL/mean": -22.344417572021484, + "KL/rejected_KL_mean": -27.532669067382812, + "KL/std": 14.876508712768555, + "epoch": 0.30990173847316704, + "fcm_dpo/beta": 0.06920944154262543, + "fcm_dpo/delta": -0.3207128643989563, + "fcm_dpo/margin": 10.37651252746582, + "fcm_dpo/q_t": 0.3630355894565582, + "grad_norm": 32.464195251464844, + "learning_rate": 4.3719511720570814e-07, + "logits/chosen": 0.40594857931137085, + "logits/rejected": 0.33821791410446167, + "logps/chosen": -87.51234436035156, + "logps/ref_chosen": -70.3561782836914, + "logps/ref_rejected": -93.39848327636719, + "logps/rejected": -120.93115234375, + "loss": 1.078, + "margin_dpo/margin_mean": 10.376511573791504, + "margin_dpo/margin_std": 17.260995864868164, + "step": 205 + }, + { + "KL/chosen_KL_mean": -18.05939483642578, + "KL/mean": -22.194053649902344, + "KL/rejected_KL_mean": -26.328716278076172, + "KL/std": 14.585685729980469, + "epoch": 0.31141345427059713, + "fcm_dpo/beta": 0.06900796294212341, + "fcm_dpo/delta": -0.17160359025001526, + "fcm_dpo/margin": 8.269315719604492, + "fcm_dpo/q_t": 0.3914671242237091, + "grad_norm": 27.170581817626953, + "learning_rate": 4.363161124189387e-07, + "logits/chosen": 0.43547555804252625, + "logits/rejected": 0.4183978736400604, + "logps/chosen": -85.70487976074219, + "logps/ref_chosen": -67.64547729492188, + "logps/ref_rejected": -79.89584350585938, + "logps/rejected": -106.22456359863281, + "loss": 1.1931, + "margin_dpo/margin_mean": 8.269315719604492, + "margin_dpo/margin_std": 17.16391372680664, + "step": 206 + }, + { + "KL/chosen_KL_mean": -20.897016525268555, + "KL/mean": -26.49903678894043, + "KL/rejected_KL_mean": -32.10105514526367, + "KL/std": 15.320511817932129, + "epoch": 0.3129251700680272, + "fcm_dpo/beta": 0.0685008317232132, + "fcm_dpo/delta": -0.3704529404640198, + "fcm_dpo/margin": 11.20404052734375, + "fcm_dpo/q_t": 0.35451504588127136, + "grad_norm": 24.268613815307617, + "learning_rate": 4.3543189596998986e-07, + "logits/chosen": 0.3424272835254669, + "logits/rejected": 0.27394890785217285, + "logps/chosen": -88.56121063232422, + "logps/ref_chosen": -67.66419219970703, + "logps/ref_rejected": -85.10249328613281, + "logps/rejected": -117.20354461669922, + "loss": 0.9895, + "margin_dpo/margin_mean": 11.20404052734375, + "margin_dpo/margin_std": 15.7493314743042, + "step": 207 + }, + { + "KL/chosen_KL_mean": -16.35006332397461, + "KL/mean": -19.338808059692383, + "KL/rejected_KL_mean": -22.32754898071289, + "KL/std": 13.229108810424805, + "epoch": 0.3144368858654573, + "fcm_dpo/beta": 0.0683794915676117, + "fcm_dpo/delta": -0.00877474993467331, + "fcm_dpo/margin": 5.977482795715332, + "fcm_dpo/q_t": 0.4189433157444, + "grad_norm": 28.9110164642334, + "learning_rate": 4.3454249259229664e-07, + "logits/chosen": 0.37438973784446716, + "logits/rejected": 0.34730657935142517, + "logps/chosen": -74.08177185058594, + "logps/ref_chosen": -57.731712341308594, + "logps/ref_rejected": -74.19276428222656, + "logps/rejected": -96.52030944824219, + "loss": 1.2623, + "margin_dpo/margin_mean": 5.977482795715332, + "margin_dpo/margin_std": 15.280378341674805, + "step": 208 + }, + { + "KL/chosen_KL_mean": -16.869646072387695, + "KL/mean": -23.2205810546875, + "KL/rejected_KL_mean": -29.57152557373047, + "KL/std": 15.852048873901367, + "epoch": 0.31594860166288735, + "fcm_dpo/beta": 0.06793908774852753, + "fcm_dpo/delta": -0.4668008089065552, + "fcm_dpo/margin": 12.701875686645508, + "fcm_dpo/q_t": 0.33582907915115356, + "grad_norm": 27.342981338500977, + "learning_rate": 4.336479271643833e-07, + "logits/chosen": 0.3405318558216095, + "logits/rejected": 0.2840922772884369, + "logps/chosen": -85.41972351074219, + "logps/ref_chosen": -68.55007934570312, + "logps/ref_rejected": -87.90541076660156, + "logps/rejected": -117.47694396972656, + "loss": 0.9967, + "margin_dpo/margin_mean": 12.701875686645508, + "margin_dpo/margin_std": 18.238468170166016, + "step": 209 + }, + { + "KL/chosen_KL_mean": -17.2132511138916, + "KL/mean": -23.413440704345703, + "KL/rejected_KL_mean": -29.613632202148438, + "KL/std": 16.748790740966797, + "epoch": 0.31746031746031744, + "fcm_dpo/beta": 0.06733535975217819, + "fcm_dpo/delta": -0.43848103284835815, + "fcm_dpo/margin": 12.400379180908203, + "fcm_dpo/q_t": 0.3437298536300659, + "grad_norm": 22.850576400756836, + "learning_rate": 4.327482247091679e-07, + "logits/chosen": 0.4188450872898102, + "logits/rejected": 0.3194931149482727, + "logps/chosen": -74.48152160644531, + "logps/ref_chosen": -57.268272399902344, + "logps/ref_rejected": -85.72807312011719, + "logps/rejected": -115.34170532226562, + "loss": 0.9847, + "margin_dpo/margin_mean": 12.400378227233887, + "margin_dpo/margin_std": 17.264799118041992, + "step": 210 + }, + { + "KL/chosen_KL_mean": -15.078107833862305, + "KL/mean": -20.734981536865234, + "KL/rejected_KL_mean": -26.391857147216797, + "KL/std": 14.261140823364258, + "epoch": 0.31897203325774753, + "fcm_dpo/beta": 0.06684620678424835, + "fcm_dpo/delta": -0.3588387370109558, + "fcm_dpo/margin": 11.31374740600586, + "fcm_dpo/q_t": 0.355030357837677, + "grad_norm": 28.313282012939453, + "learning_rate": 4.3184341039326217e-07, + "logits/chosen": 0.42388325929641724, + "logits/rejected": 0.3282184600830078, + "logps/chosen": -68.71881866455078, + "logps/ref_chosen": -53.640708923339844, + "logps/ref_rejected": -93.0387954711914, + "logps/rejected": -119.43064880371094, + "loss": 0.9752, + "margin_dpo/margin_mean": 11.31374740600586, + "margin_dpo/margin_std": 15.375458717346191, + "step": 211 + }, + { + "KL/chosen_KL_mean": -17.590024948120117, + "KL/mean": -23.55065155029297, + "KL/rejected_KL_mean": -29.51128387451172, + "KL/std": 15.991048812866211, + "epoch": 0.3204837490551776, + "fcm_dpo/beta": 0.06628412753343582, + "fcm_dpo/delta": -0.3932895362377167, + "fcm_dpo/margin": 11.921259880065918, + "fcm_dpo/q_t": 0.34739983081817627, + "grad_norm": 23.193771362304688, + "learning_rate": 4.309335095262675e-07, + "logits/chosen": 0.4299660921096802, + "logits/rejected": 0.35489726066589355, + "logps/chosen": -74.95677185058594, + "logps/ref_chosen": -57.36674499511719, + "logps/ref_rejected": -79.89643096923828, + "logps/rejected": -109.40771484375, + "loss": 0.9986, + "margin_dpo/margin_mean": 11.921259880065918, + "margin_dpo/margin_std": 16.89865493774414, + "step": 212 + }, + { + "KL/chosen_KL_mean": -13.309558868408203, + "KL/mean": -20.039276123046875, + "KL/rejected_KL_mean": -26.76898956298828, + "KL/std": 16.082687377929688, + "epoch": 0.3219954648526077, + "fcm_dpo/beta": 0.06563502550125122, + "fcm_dpo/delta": -0.4880368709564209, + "fcm_dpo/margin": 13.459427833557129, + "fcm_dpo/q_t": 0.3337309658527374, + "grad_norm": 21.686983108520508, + "learning_rate": 4.3001854756006724e-07, + "logits/chosen": 0.37352073192596436, + "logits/rejected": 0.34911400079727173, + "logps/chosen": -78.53067016601562, + "logps/ref_chosen": -65.22111511230469, + "logps/ref_rejected": -80.1810302734375, + "logps/rejected": -106.95001220703125, + "loss": 0.9682, + "margin_dpo/margin_mean": 13.459428787231445, + "margin_dpo/margin_std": 18.411163330078125, + "step": 213 + }, + { + "KL/chosen_KL_mean": -15.645334243774414, + "KL/mean": -22.310592651367188, + "KL/rejected_KL_mean": -28.97585105895996, + "KL/std": 16.025737762451172, + "epoch": 0.3235071806500378, + "fcm_dpo/beta": 0.06511686742305756, + "fcm_dpo/delta": -0.4718652665615082, + "fcm_dpo/margin": 13.330512046813965, + "fcm_dpo/q_t": 0.3404635190963745, + "grad_norm": 27.284984588623047, + "learning_rate": 4.290985500881143e-07, + "logits/chosen": 0.2772884964942932, + "logits/rejected": 0.2548210024833679, + "logps/chosen": -76.93766784667969, + "logps/ref_chosen": -61.292327880859375, + "logps/ref_rejected": -67.69841003417969, + "logps/rejected": -96.67425537109375, + "loss": 0.9785, + "margin_dpo/margin_mean": 13.330513000488281, + "margin_dpo/margin_std": 18.1436767578125, + "step": 214 + }, + { + "KL/chosen_KL_mean": -18.202037811279297, + "KL/mean": -25.111045837402344, + "KL/rejected_KL_mean": -32.020050048828125, + "KL/std": 16.583818435668945, + "epoch": 0.3250188964474679, + "fcm_dpo/beta": 0.06448207795619965, + "fcm_dpo/delta": -0.4951305389404297, + "fcm_dpo/margin": 13.818011283874512, + "fcm_dpo/q_t": 0.33746108412742615, + "grad_norm": 24.17980194091797, + "learning_rate": 4.281735428447157e-07, + "logits/chosen": 0.30728620290756226, + "logits/rejected": 0.19806616008281708, + "logps/chosen": -82.07117462158203, + "logps/ref_chosen": -63.869136810302734, + "logps/ref_rejected": -98.7657241821289, + "logps/rejected": -130.7857666015625, + "loss": 0.965, + "margin_dpo/margin_mean": 13.818011283874512, + "margin_dpo/margin_std": 18.296720504760742, + "step": 215 + }, + { + "KL/chosen_KL_mean": -16.047607421875, + "KL/mean": -23.113426208496094, + "KL/rejected_KL_mean": -30.179241180419922, + "KL/std": 16.211938858032227, + "epoch": 0.32653061224489793, + "fcm_dpo/beta": 0.06376518309116364, + "fcm_dpo/delta": -0.5057007670402527, + "fcm_dpo/margin": 14.131632804870605, + "fcm_dpo/q_t": 0.3280887007713318, + "grad_norm": 23.373916625976562, + "learning_rate": 4.2724355170431247e-07, + "logits/chosen": 0.4402976632118225, + "logits/rejected": 0.3519246578216553, + "logps/chosen": -83.87255859375, + "logps/ref_chosen": -67.824951171875, + "logps/ref_rejected": -96.40231323242188, + "logps/rejected": -126.58155822753906, + "loss": 0.9157, + "margin_dpo/margin_mean": 14.131633758544922, + "margin_dpo/margin_std": 17.458927154541016, + "step": 216 + }, + { + "KL/chosen_KL_mean": -17.97003936767578, + "KL/mean": -25.500181198120117, + "KL/rejected_KL_mean": -33.03032684326172, + "KL/std": 17.082275390625, + "epoch": 0.328042328042328, + "fcm_dpo/beta": 0.0630604475736618, + "fcm_dpo/delta": -0.5551861524581909, + "fcm_dpo/margin": 15.060293197631836, + "fcm_dpo/q_t": 0.3233886957168579, + "grad_norm": 21.426084518432617, + "learning_rate": 4.26308602680756e-07, + "logits/chosen": 0.33799299597740173, + "logits/rejected": 0.2368866205215454, + "logps/chosen": -78.47503662109375, + "logps/ref_chosen": -60.5049934387207, + "logps/ref_rejected": -84.26618194580078, + "logps/rejected": -117.2965087890625, + "loss": 0.8898, + "margin_dpo/margin_mean": 15.060293197631836, + "margin_dpo/margin_std": 18.075111389160156, + "step": 217 + }, + { + "KL/chosen_KL_mean": -19.045536041259766, + "KL/mean": -23.663829803466797, + "KL/rejected_KL_mean": -28.282123565673828, + "KL/std": 15.749072074890137, + "epoch": 0.3295540438397581, + "fcm_dpo/beta": 0.06281279027462006, + "fcm_dpo/delta": -0.181310772895813, + "fcm_dpo/margin": 9.236587524414062, + "fcm_dpo/q_t": 0.3881898522377014, + "grad_norm": 25.730531692504883, + "learning_rate": 4.253687219265803e-07, + "logits/chosen": 0.27604031562805176, + "logits/rejected": 0.2712269723415375, + "logps/chosen": -89.63984680175781, + "logps/ref_chosen": -70.59431457519531, + "logps/ref_rejected": -73.89038848876953, + "logps/rejected": -102.17251586914062, + "loss": 1.213, + "margin_dpo/margin_mean": 9.236588478088379, + "margin_dpo/margin_std": 19.432811737060547, + "step": 218 + }, + { + "KL/chosen_KL_mean": -17.510616302490234, + "KL/mean": -22.187911987304688, + "KL/rejected_KL_mean": -26.86521339416504, + "KL/std": 16.233766555786133, + "epoch": 0.3310657596371882, + "fcm_dpo/beta": 0.062476158142089844, + "fcm_dpo/delta": -0.18564411997795105, + "fcm_dpo/margin": 9.354594230651855, + "fcm_dpo/q_t": 0.38181036710739136, + "grad_norm": 23.56931495666504, + "learning_rate": 4.2442393573227043e-07, + "logits/chosen": 0.34160101413726807, + "logits/rejected": 0.29895228147506714, + "logps/chosen": -78.00155639648438, + "logps/ref_chosen": -60.490943908691406, + "logps/ref_rejected": -75.85001373291016, + "logps/rejected": -102.71522521972656, + "loss": 1.0854, + "margin_dpo/margin_mean": 9.354593276977539, + "margin_dpo/margin_std": 15.447464942932129, + "step": 219 + }, + { + "KL/chosen_KL_mean": -14.94304084777832, + "KL/mean": -20.628089904785156, + "KL/rejected_KL_mean": -26.313140869140625, + "KL/std": 16.361404418945312, + "epoch": 0.3325774754346183, + "fcm_dpo/beta": 0.0620996467769146, + "fcm_dpo/delta": -0.30871567130088806, + "fcm_dpo/margin": 11.370096206665039, + "fcm_dpo/q_t": 0.3682330548763275, + "grad_norm": 22.7717227935791, + "learning_rate": 4.234742705255272e-07, + "logits/chosen": 0.4229661524295807, + "logits/rejected": 0.35844409465789795, + "logps/chosen": -59.95643997192383, + "logps/ref_chosen": -45.013397216796875, + "logps/ref_rejected": -70.49369812011719, + "logps/rejected": -96.80683898925781, + "loss": 1.0622, + "margin_dpo/margin_mean": 11.370097160339355, + "margin_dpo/margin_std": 18.118942260742188, + "step": 220 + }, + { + "KL/chosen_KL_mean": -14.233732223510742, + "KL/mean": -20.45446014404297, + "KL/rejected_KL_mean": -26.675186157226562, + "KL/std": 16.154266357421875, + "epoch": 0.3340891912320484, + "fcm_dpo/beta": 0.06174083054065704, + "fcm_dpo/delta": -0.3708839416503906, + "fcm_dpo/margin": 12.441455841064453, + "fcm_dpo/q_t": 0.3594486117362976, + "grad_norm": 24.052045822143555, + "learning_rate": 4.22519752870528e-07, + "logits/chosen": 0.40941134095191956, + "logits/rejected": 0.3343515992164612, + "logps/chosen": -73.32957458496094, + "logps/ref_chosen": -59.09584045410156, + "logps/ref_rejected": -88.64388275146484, + "logps/rejected": -115.3190689086914, + "loss": 1.0153, + "margin_dpo/margin_mean": 12.441455841064453, + "margin_dpo/margin_std": 18.81906509399414, + "step": 221 + }, + { + "KL/chosen_KL_mean": -14.926090240478516, + "KL/mean": -23.0926456451416, + "KL/rejected_KL_mean": -31.259204864501953, + "KL/std": 18.31246566772461, + "epoch": 0.3356009070294785, + "fcm_dpo/beta": 0.061059437692165375, + "fcm_dpo/delta": -0.6032091379165649, + "fcm_dpo/margin": 16.333114624023438, + "fcm_dpo/q_t": 0.3135583996772766, + "grad_norm": 21.939809799194336, + "learning_rate": 4.2156040946718343e-07, + "logits/chosen": 0.45339035987854004, + "logits/rejected": 0.36432236433029175, + "logps/chosen": -70.92378234863281, + "logps/ref_chosen": -55.9976921081543, + "logps/ref_rejected": -111.94727325439453, + "logps/rejected": -143.20648193359375, + "loss": 0.8709, + "margin_dpo/margin_mean": 16.333114624023438, + "margin_dpo/margin_std": 18.66763687133789, + "step": 222 + }, + { + "KL/chosen_KL_mean": -16.65311050415039, + "KL/mean": -23.59942626953125, + "KL/rejected_KL_mean": -30.545745849609375, + "KL/std": 17.56645393371582, + "epoch": 0.3371126228269085, + "fcm_dpo/beta": 0.060423314571380615, + "fcm_dpo/delta": -0.4435126781463623, + "fcm_dpo/margin": 13.8926420211792, + "fcm_dpo/q_t": 0.3385217785835266, + "grad_norm": 21.672243118286133, + "learning_rate": 4.2059626715039065e-07, + "logits/chosen": 0.42599231004714966, + "logits/rejected": 0.36808210611343384, + "logps/chosen": -76.5445327758789, + "logps/ref_chosen": -59.891422271728516, + "logps/ref_rejected": -86.28954315185547, + "logps/rejected": -116.83528900146484, + "loss": 0.9288, + "margin_dpo/margin_mean": 13.892641067504883, + "margin_dpo/margin_std": 17.020523071289062, + "step": 223 + }, + { + "KL/chosen_KL_mean": -18.70929718017578, + "KL/mean": -22.799972534179688, + "KL/rejected_KL_mean": -26.890644073486328, + "KL/std": 16.89524269104004, + "epoch": 0.3386243386243386, + "fcm_dpo/beta": 0.060215141624212265, + "fcm_dpo/delta": -0.09318163245916367, + "fcm_dpo/margin": 8.18134880065918, + "fcm_dpo/q_t": 0.40444496273994446, + "grad_norm": 28.343555450439453, + "learning_rate": 4.1962735288928304e-07, + "logits/chosen": 0.48086220026016235, + "logits/rejected": 0.45792585611343384, + "logps/chosen": -82.75393676757812, + "logps/ref_chosen": -64.04463195800781, + "logps/ref_rejected": -75.05450439453125, + "logps/rejected": -101.94514465332031, + "loss": 1.1685, + "margin_dpo/margin_mean": 8.18134880065918, + "margin_dpo/margin_std": 16.779151916503906, + "step": 224 + }, + { + "KL/chosen_KL_mean": -18.021066665649414, + "KL/mean": -25.016357421875, + "KL/rejected_KL_mean": -32.01164627075195, + "KL/std": 19.494888305664062, + "epoch": 0.3401360544217687, + "fcm_dpo/beta": 0.05976982042193413, + "fcm_dpo/delta": -0.440379798412323, + "fcm_dpo/margin": 13.990575790405273, + "fcm_dpo/q_t": 0.3423454761505127, + "grad_norm": 26.42709732055664, + "learning_rate": 4.186536937864752e-07, + "logits/chosen": 0.4376928508281708, + "logits/rejected": 0.3216109871864319, + "logps/chosen": -84.11688995361328, + "logps/ref_chosen": -66.0958251953125, + "logps/ref_rejected": -97.68675231933594, + "logps/rejected": -129.69839477539062, + "loss": 0.956, + "margin_dpo/margin_mean": 13.990575790405273, + "margin_dpo/margin_std": 18.345081329345703, + "step": 225 + }, + { + "KL/chosen_KL_mean": -15.736995697021484, + "KL/mean": -21.353412628173828, + "KL/rejected_KL_mean": -26.969825744628906, + "KL/std": 17.166458129882812, + "epoch": 0.3416477702191988, + "fcm_dpo/beta": 0.05945456773042679, + "fcm_dpo/delta": -0.26960688829421997, + "fcm_dpo/margin": 11.232826232910156, + "fcm_dpo/q_t": 0.36949968338012695, + "grad_norm": 22.89041519165039, + "learning_rate": 4.176753170773052e-07, + "logits/chosen": 0.4460662603378296, + "logits/rejected": 0.39804208278656006, + "logps/chosen": -67.15386962890625, + "logps/ref_chosen": -51.4168701171875, + "logps/ref_rejected": -66.30068969726562, + "logps/rejected": -93.2705078125, + "loss": 1.0926, + "margin_dpo/margin_mean": 11.23282527923584, + "margin_dpo/margin_std": 18.93710708618164, + "step": 226 + }, + { + "KL/chosen_KL_mean": -16.629560470581055, + "KL/mean": -22.549894332885742, + "KL/rejected_KL_mean": -28.470233917236328, + "KL/std": 18.3953857421875, + "epoch": 0.3431594860166289, + "fcm_dpo/beta": 0.05908646434545517, + "fcm_dpo/delta": -0.3017960786819458, + "fcm_dpo/margin": 11.84067153930664, + "fcm_dpo/q_t": 0.36926305294036865, + "grad_norm": 26.52588653564453, + "learning_rate": 4.166922501290729e-07, + "logits/chosen": 0.4893955886363983, + "logits/rejected": 0.4483721852302551, + "logps/chosen": -74.61933898925781, + "logps/ref_chosen": -57.989776611328125, + "logps/ref_rejected": -75.05464172363281, + "logps/rejected": -103.5248794555664, + "loss": 1.0907, + "margin_dpo/margin_mean": 11.84067153930664, + "margin_dpo/margin_std": 20.150123596191406, + "step": 227 + }, + { + "KL/chosen_KL_mean": -17.562301635742188, + "KL/mean": -23.8963680267334, + "KL/rejected_KL_mean": -30.230426788330078, + "KL/std": 17.28730010986328, + "epoch": 0.34467120181405897, + "fcm_dpo/beta": 0.05873169004917145, + "fcm_dpo/delta": -0.3464760482311249, + "fcm_dpo/margin": 12.668121337890625, + "fcm_dpo/q_t": 0.3563549816608429, + "grad_norm": 21.39133071899414, + "learning_rate": 4.1570452044027405e-07, + "logits/chosen": 0.4631214141845703, + "logits/rejected": 0.3819465637207031, + "logps/chosen": -73.12167358398438, + "logps/ref_chosen": -55.55936813354492, + "logps/ref_rejected": -77.02364349365234, + "logps/rejected": -107.25407409667969, + "loss": 1.019, + "margin_dpo/margin_mean": 12.668121337890625, + "margin_dpo/margin_std": 18.546981811523438, + "step": 228 + }, + { + "KL/chosen_KL_mean": -14.690620422363281, + "KL/mean": -20.11794662475586, + "KL/rejected_KL_mean": -25.545269012451172, + "KL/std": 16.276611328125, + "epoch": 0.34618291761148906, + "fcm_dpo/beta": 0.05839953571557999, + "fcm_dpo/delta": -0.23537826538085938, + "fcm_dpo/margin": 10.854650497436523, + "fcm_dpo/q_t": 0.3731439411640167, + "grad_norm": 48.165771484375, + "learning_rate": 4.147121556398312e-07, + "logits/chosen": 0.518801212310791, + "logits/rejected": 0.44911321997642517, + "logps/chosen": -65.48528289794922, + "logps/ref_chosen": -50.79466247558594, + "logps/ref_rejected": -78.4474105834961, + "logps/rejected": -103.99267578125, + "loss": 1.119, + "margin_dpo/margin_mean": 10.854650497436523, + "margin_dpo/margin_std": 19.43063735961914, + "step": 229 + }, + { + "KL/chosen_KL_mean": -16.100372314453125, + "KL/mean": -22.5364990234375, + "KL/rejected_KL_mean": -28.972625732421875, + "KL/std": 18.35205078125, + "epoch": 0.3476946334089191, + "fcm_dpo/beta": 0.058091744780540466, + "fcm_dpo/delta": -0.3502655327320099, + "fcm_dpo/margin": 12.8722562789917, + "fcm_dpo/q_t": 0.3544079661369324, + "grad_norm": 22.67949867248535, + "learning_rate": 4.137151834863213e-07, + "logits/chosen": 0.43540120124816895, + "logits/rejected": 0.4321771562099457, + "logps/chosen": -72.82959747314453, + "logps/ref_chosen": -56.729225158691406, + "logps/ref_rejected": -62.99180603027344, + "logps/rejected": -91.96443176269531, + "loss": 1.0035, + "margin_dpo/margin_mean": 12.872257232666016, + "margin_dpo/margin_std": 18.22582244873047, + "step": 230 + }, + { + "KL/chosen_KL_mean": -17.891250610351562, + "KL/mean": -27.22870445251465, + "KL/rejected_KL_mean": -36.566162109375, + "KL/std": 17.813358306884766, + "epoch": 0.3492063492063492, + "fcm_dpo/beta": 0.05730242654681206, + "fcm_dpo/delta": -0.6777905225753784, + "fcm_dpo/margin": 18.674917221069336, + "fcm_dpo/q_t": 0.29342564940452576, + "grad_norm": 22.191556930541992, + "learning_rate": 4.1271363186719835e-07, + "logits/chosen": 0.3374232053756714, + "logits/rejected": 0.32494157552719116, + "logps/chosen": -90.48834991455078, + "logps/ref_chosen": -72.59709930419922, + "logps/ref_rejected": -86.2322998046875, + "logps/rejected": -122.7984619140625, + "loss": 0.8052, + "margin_dpo/margin_mean": 18.674915313720703, + "margin_dpo/margin_std": 18.372806549072266, + "step": 231 + }, + { + "KL/chosen_KL_mean": -18.077415466308594, + "KL/mean": -24.293066024780273, + "KL/rejected_KL_mean": -30.508716583251953, + "KL/std": 19.465444564819336, + "epoch": 0.3507180650037793, + "fcm_dpo/beta": 0.05685997009277344, + "fcm_dpo/delta": -0.3091171383857727, + "fcm_dpo/margin": 12.431302070617676, + "fcm_dpo/q_t": 0.37119412422180176, + "grad_norm": 27.450639724731445, + "learning_rate": 4.1170752879801436e-07, + "logits/chosen": 0.3792431950569153, + "logits/rejected": 0.3492290675640106, + "logps/chosen": -86.1959457397461, + "logps/ref_chosen": -68.1185302734375, + "logps/ref_rejected": -83.79415893554688, + "logps/rejected": -114.3028793334961, + "loss": 1.1051, + "margin_dpo/margin_mean": 12.431303024291992, + "margin_dpo/margin_std": 22.162633895874023, + "step": 232 + }, + { + "KL/chosen_KL_mean": -20.86438751220703, + "KL/mean": -25.95412826538086, + "KL/rejected_KL_mean": -31.043869018554688, + "KL/std": 18.983388900756836, + "epoch": 0.35222978080120937, + "fcm_dpo/beta": 0.05648782104253769, + "fcm_dpo/delta": -0.2631673514842987, + "fcm_dpo/margin": 10.17947769165039, + "fcm_dpo/q_t": 0.3917330205440521, + "grad_norm": 23.20311164855957, + "learning_rate": 4.106969024216348e-07, + "logits/chosen": 0.4511150121688843, + "logits/rejected": 0.39118489623069763, + "logps/chosen": -75.93453979492188, + "logps/ref_chosen": -55.070152282714844, + "logps/ref_rejected": -66.61845397949219, + "logps/rejected": -97.66232299804688, + "loss": 1.1335, + "margin_dpo/margin_mean": 10.17947769165039, + "margin_dpo/margin_std": 18.150503158569336, + "step": 233 + }, + { + "KL/chosen_KL_mean": -18.357402801513672, + "KL/mean": -22.90506935119629, + "KL/rejected_KL_mean": -27.452739715576172, + "KL/std": 17.39879035949707, + "epoch": 0.35374149659863946, + "fcm_dpo/beta": 0.05636388435959816, + "fcm_dpo/delta": -0.21988779306411743, + "fcm_dpo/margin": 9.095338821411133, + "fcm_dpo/q_t": 0.40208858251571655, + "grad_norm": 26.619951248168945, + "learning_rate": 4.09681781007452e-07, + "logits/chosen": 0.3563153147697449, + "logits/rejected": 0.34465306997299194, + "logps/chosen": -74.28329467773438, + "logps/ref_chosen": -55.92589569091797, + "logps/ref_rejected": -51.11608123779297, + "logps/rejected": -78.56881713867188, + "loss": 1.2088, + "margin_dpo/margin_mean": 9.095338821411133, + "margin_dpo/margin_std": 19.057300567626953, + "step": 234 + }, + { + "KL/chosen_KL_mean": -14.86567497253418, + "KL/mean": -23.364036560058594, + "KL/rejected_KL_mean": -31.862403869628906, + "KL/std": 19.081268310546875, + "epoch": 0.35525321239606955, + "fcm_dpo/beta": 0.05578460916876793, + "fcm_dpo/delta": -0.5533231496810913, + "fcm_dpo/margin": 16.99673080444336, + "fcm_dpo/q_t": 0.3123379349708557, + "grad_norm": 20.249174118041992, + "learning_rate": 4.08662192950594e-07, + "logits/chosen": 0.4756525754928589, + "logits/rejected": 0.4570575952529907, + "logps/chosen": -79.4053955078125, + "logps/ref_chosen": -64.53972625732422, + "logps/ref_rejected": -77.69151306152344, + "logps/rejected": -109.55390930175781, + "loss": 0.831, + "margin_dpo/margin_mean": 16.99673080444336, + "margin_dpo/margin_std": 17.240177154541016, + "step": 235 + }, + { + "KL/chosen_KL_mean": -23.86891746520996, + "KL/mean": -30.413143157958984, + "KL/rejected_KL_mean": -36.95737075805664, + "KL/std": 19.1519775390625, + "epoch": 0.35676492819349964, + "fcm_dpo/beta": 0.05531569570302963, + "fcm_dpo/delta": -0.3266342282295227, + "fcm_dpo/margin": 13.08845329284668, + "fcm_dpo/q_t": 0.3606201708316803, + "grad_norm": 24.561580657958984, + "learning_rate": 4.076381667711306e-07, + "logits/chosen": 0.3800908923149109, + "logits/rejected": 0.366105854511261, + "logps/chosen": -95.02365112304688, + "logps/ref_chosen": -71.15473937988281, + "logps/ref_rejected": -84.88541412353516, + "logps/rejected": -121.84278869628906, + "loss": 1.0543, + "margin_dpo/margin_mean": 13.088452339172363, + "margin_dpo/margin_std": 20.483673095703125, + "step": 236 + }, + { + "KL/chosen_KL_mean": -19.124282836914062, + "KL/mean": -26.505462646484375, + "KL/rejected_KL_mean": -33.88664245605469, + "KL/std": 18.42672348022461, + "epoch": 0.35827664399092973, + "fcm_dpo/beta": 0.05492132902145386, + "fcm_dpo/delta": -0.4140962064266205, + "fcm_dpo/margin": 14.762357711791992, + "fcm_dpo/q_t": 0.33958035707473755, + "grad_norm": 26.182437896728516, + "learning_rate": 4.066097311132753e-07, + "logits/chosen": 0.47192585468292236, + "logits/rejected": 0.4588091969490051, + "logps/chosen": -95.26629638671875, + "logps/ref_chosen": -76.14201354980469, + "logps/ref_rejected": -80.88479614257812, + "logps/rejected": -114.77143859863281, + "loss": 1.0072, + "margin_dpo/margin_mean": 14.762357711791992, + "margin_dpo/margin_std": 20.582250595092773, + "step": 237 + }, + { + "KL/chosen_KL_mean": -17.897552490234375, + "KL/mean": -25.2412109375, + "KL/rejected_KL_mean": -32.584869384765625, + "KL/std": 19.855119705200195, + "epoch": 0.35978835978835977, + "fcm_dpo/beta": 0.05443461239337921, + "fcm_dpo/delta": -0.403104305267334, + "fcm_dpo/margin": 14.687314987182617, + "fcm_dpo/q_t": 0.34872615337371826, + "grad_norm": 30.09897232055664, + "learning_rate": 4.0557691474458414e-07, + "logits/chosen": 0.40769416093826294, + "logits/rejected": 0.39426884055137634, + "logps/chosen": -86.78240203857422, + "logps/ref_chosen": -68.88484954833984, + "logps/ref_rejected": -75.8946304321289, + "logps/rejected": -108.47949981689453, + "loss": 0.9901, + "margin_dpo/margin_mean": 14.68731689453125, + "margin_dpo/margin_std": 20.21993637084961, + "step": 238 + }, + { + "KL/chosen_KL_mean": -22.819931030273438, + "KL/mean": -30.486026763916016, + "KL/rejected_KL_mean": -38.152122497558594, + "KL/std": 21.54601287841797, + "epoch": 0.36130007558578986, + "fcm_dpo/beta": 0.05407198518514633, + "fcm_dpo/delta": -0.4323962330818176, + "fcm_dpo/margin": 15.332193374633789, + "fcm_dpo/q_t": 0.3422353267669678, + "grad_norm": 24.095972061157227, + "learning_rate": 4.045397465551513e-07, + "logits/chosen": 0.5657989978790283, + "logits/rejected": 0.4331953227519989, + "logps/chosen": -79.59175872802734, + "logps/ref_chosen": -56.771827697753906, + "logps/ref_rejected": -116.23050689697266, + "logps/rejected": -154.38262939453125, + "loss": 0.9827, + "margin_dpo/margin_mean": 15.332193374633789, + "margin_dpo/margin_std": 20.919029235839844, + "step": 239 + }, + { + "KL/chosen_KL_mean": -21.129817962646484, + "KL/mean": -30.673057556152344, + "KL/rejected_KL_mean": -40.2162971496582, + "KL/std": 22.056163787841797, + "epoch": 0.36281179138321995, + "fcm_dpo/beta": 0.05341993644833565, + "fcm_dpo/delta": -0.625713586807251, + "fcm_dpo/margin": 19.086475372314453, + "fcm_dpo/q_t": 0.30604344606399536, + "grad_norm": 17.801057815551758, + "learning_rate": 4.0349825555680045e-07, + "logits/chosen": 0.44967496395111084, + "logits/rejected": 0.3539752960205078, + "logps/chosen": -74.48393249511719, + "logps/ref_chosen": -53.35411071777344, + "logps/ref_rejected": -80.12019348144531, + "logps/rejected": -120.33649444580078, + "loss": 0.8536, + "margin_dpo/margin_mean": 19.086475372314453, + "margin_dpo/margin_std": 20.889379501342773, + "step": 240 + }, + { + "KL/chosen_KL_mean": -20.96868896484375, + "KL/mean": -26.57484245300293, + "KL/rejected_KL_mean": -32.180992126464844, + "KL/std": 19.14997100830078, + "epoch": 0.36432350718065004, + "fcm_dpo/beta": 0.053084079176187515, + "fcm_dpo/delta": -0.19632312655448914, + "fcm_dpo/margin": 11.212306022644043, + "fcm_dpo/q_t": 0.3864910900592804, + "grad_norm": 25.237791061401367, + "learning_rate": 4.0245247088227377e-07, + "logits/chosen": 0.4189719557762146, + "logits/rejected": 0.3810345232486725, + "logps/chosen": -92.86410522460938, + "logps/ref_chosen": -71.89541625976562, + "logps/ref_rejected": -83.03492736816406, + "logps/rejected": -115.2159194946289, + "loss": 1.1167, + "margin_dpo/margin_mean": 11.21230697631836, + "margin_dpo/margin_std": 20.098365783691406, + "step": 241 + }, + { + "KL/chosen_KL_mean": -21.04944610595703, + "KL/mean": -29.4080753326416, + "KL/rejected_KL_mean": -37.76670455932617, + "KL/std": 21.98519515991211, + "epoch": 0.36583522297808013, + "fcm_dpo/beta": 0.052576761692762375, + "fcm_dpo/delta": -0.48393750190734863, + "fcm_dpo/margin": 16.717262268066406, + "fcm_dpo/q_t": 0.3379758596420288, + "grad_norm": 20.783466339111328, + "learning_rate": 4.0140242178441665e-07, + "logits/chosen": 0.41847819089889526, + "logits/rejected": 0.39497095346450806, + "logps/chosen": -78.97688293457031, + "logps/ref_chosen": -57.927433013916016, + "logps/ref_rejected": -67.838623046875, + "logps/rejected": -105.6053237915039, + "loss": 0.9692, + "margin_dpo/margin_mean": 16.717260360717773, + "margin_dpo/margin_std": 22.970556259155273, + "step": 242 + }, + { + "KL/chosen_KL_mean": -20.54830551147461, + "KL/mean": -28.065828323364258, + "KL/rejected_KL_mean": -35.58334732055664, + "KL/std": 20.905529022216797, + "epoch": 0.3673469387755102, + "fcm_dpo/beta": 0.05220145732164383, + "fcm_dpo/delta": -0.3878943920135498, + "fcm_dpo/margin": 15.035041809082031, + "fcm_dpo/q_t": 0.3498421311378479, + "grad_norm": 22.502704620361328, + "learning_rate": 4.003481376353596e-07, + "logits/chosen": 0.4004201889038086, + "logits/rejected": 0.40324679017066956, + "logps/chosen": -94.82498168945312, + "logps/ref_chosen": -74.27667236328125, + "logps/ref_rejected": -73.24340057373047, + "logps/rejected": -108.82675170898438, + "loss": 0.9925, + "margin_dpo/margin_mean": 15.035041809082031, + "margin_dpo/margin_std": 20.981693267822266, + "step": 243 + }, + { + "KL/chosen_KL_mean": -20.757890701293945, + "KL/mean": -30.753223419189453, + "KL/rejected_KL_mean": -40.748558044433594, + "KL/std": 20.72345733642578, + "epoch": 0.3688586545729403, + "fcm_dpo/beta": 0.051632143557071686, + "fcm_dpo/delta": -0.6384683847427368, + "fcm_dpo/margin": 19.990671157836914, + "fcm_dpo/q_t": 0.30057188868522644, + "grad_norm": 18.70012664794922, + "learning_rate": 3.9928964792569654e-07, + "logits/chosen": 0.4833059310913086, + "logits/rejected": 0.3956766128540039, + "logps/chosen": -74.12179565429688, + "logps/ref_chosen": -53.36390686035156, + "logps/ref_rejected": -71.10276794433594, + "logps/rejected": -111.85133361816406, + "loss": 0.8098, + "margin_dpo/margin_mean": 19.99066925048828, + "margin_dpo/margin_std": 20.109310150146484, + "step": 244 + }, + { + "KL/chosen_KL_mean": -21.923555374145508, + "KL/mean": -33.13026428222656, + "KL/rejected_KL_mean": -44.336971282958984, + "KL/std": 21.946651458740234, + "epoch": 0.37037037037037035, + "fcm_dpo/beta": 0.05085434392094612, + "fcm_dpo/delta": -0.7485265731811523, + "fcm_dpo/margin": 22.413414001464844, + "fcm_dpo/q_t": 0.2832839787006378, + "grad_norm": 21.613670349121094, + "learning_rate": 3.982269822636601e-07, + "logits/chosen": 0.4945005774497986, + "logits/rejected": 0.46392127871513367, + "logps/chosen": -93.11865997314453, + "logps/ref_chosen": -71.19510650634766, + "logps/ref_rejected": -80.76235961914062, + "logps/rejected": -125.09933471679688, + "loss": 0.7623, + "margin_dpo/margin_mean": 22.413414001464844, + "margin_dpo/margin_std": 21.243215560913086, + "step": 245 + }, + { + "KL/chosen_KL_mean": -28.08251190185547, + "KL/mean": -36.99290466308594, + "KL/rejected_KL_mean": -45.90330123901367, + "KL/std": 22.29575538635254, + "epoch": 0.37188208616780044, + "fcm_dpo/beta": 0.05032268166542053, + "fcm_dpo/delta": -0.5010740756988525, + "fcm_dpo/margin": 17.82078742980957, + "fcm_dpo/q_t": 0.3402974009513855, + "grad_norm": 26.157474517822266, + "learning_rate": 3.971601703742932e-07, + "logits/chosen": 0.5023146867752075, + "logits/rejected": 0.43962740898132324, + "logps/chosen": -99.70355987548828, + "logps/ref_chosen": -71.62104797363281, + "logps/ref_rejected": -94.03392028808594, + "logps/rejected": -139.93722534179688, + "loss": 1.0338, + "margin_dpo/margin_mean": 17.82078742980957, + "margin_dpo/margin_std": 27.0596866607666, + "step": 246 + }, + { + "KL/chosen_KL_mean": -29.497520446777344, + "KL/mean": -34.97264862060547, + "KL/rejected_KL_mean": -40.447776794433594, + "KL/std": 20.803512573242188, + "epoch": 0.37339380196523053, + "fcm_dpo/beta": 0.0501084178686142, + "fcm_dpo/delta": -0.14957939088344574, + "fcm_dpo/margin": 10.950252532958984, + "fcm_dpo/q_t": 0.3890780210494995, + "grad_norm": 25.837060928344727, + "learning_rate": 3.960892420986177e-07, + "logits/chosen": 0.48695895075798035, + "logits/rejected": 0.47598302364349365, + "logps/chosen": -109.52006530761719, + "logps/ref_chosen": -80.02254486083984, + "logps/ref_rejected": -89.22705841064453, + "logps/rejected": -129.67483520507812, + "loss": 1.1687, + "margin_dpo/margin_mean": 10.950252532958984, + "margin_dpo/margin_std": 21.486289978027344, + "step": 247 + }, + { + "KL/chosen_KL_mean": -26.288965225219727, + "KL/mean": -35.254913330078125, + "KL/rejected_KL_mean": -44.22086715698242, + "KL/std": 24.730310440063477, + "epoch": 0.3749055177626606, + "fcm_dpo/beta": 0.04967677220702171, + "fcm_dpo/delta": -0.4950011074542999, + "fcm_dpo/margin": 17.93189811706543, + "fcm_dpo/q_t": 0.3469792604446411, + "grad_norm": 27.166921615600586, + "learning_rate": 3.9501422739279953e-07, + "logits/chosen": 0.4623529613018036, + "logits/rejected": 0.5076808929443359, + "logps/chosen": -91.66692352294922, + "logps/ref_chosen": -65.37796020507812, + "logps/ref_rejected": -61.365787506103516, + "logps/rejected": -105.58665466308594, + "loss": 1.0333, + "margin_dpo/margin_mean": 17.93189811706543, + "margin_dpo/margin_std": 27.281169891357422, + "step": 248 + }, + { + "KL/chosen_KL_mean": -32.922271728515625, + "KL/mean": -36.729530334472656, + "KL/rejected_KL_mean": -40.53678512573242, + "KL/std": 22.503341674804688, + "epoch": 0.3764172335600907, + "fcm_dpo/beta": 0.049469586461782455, + "fcm_dpo/delta": -0.11483334749937057, + "fcm_dpo/margin": 7.6145148277282715, + "fcm_dpo/q_t": 0.4242667555809021, + "grad_norm": 33.79492950439453, + "learning_rate": 3.9393515632731094e-07, + "logits/chosen": 0.4738476574420929, + "logits/rejected": 0.5094689130783081, + "logps/chosen": -107.52372741699219, + "logps/ref_chosen": -74.60145568847656, + "logps/ref_rejected": -63.79338455200195, + "logps/rejected": -104.33016967773438, + "loss": 1.3636, + "margin_dpo/margin_mean": 7.61451530456543, + "margin_dpo/margin_std": 23.663902282714844, + "step": 249 + }, + { + "KL/chosen_KL_mean": -28.430173873901367, + "KL/mean": -38.1130485534668, + "KL/rejected_KL_mean": -47.795928955078125, + "KL/std": 24.80286407470703, + "epoch": 0.3779289493575208, + "fcm_dpo/beta": 0.04904735088348389, + "fcm_dpo/delta": -0.5547791123390198, + "fcm_dpo/margin": 19.365753173828125, + "fcm_dpo/q_t": 0.32286572456359863, + "grad_norm": 22.701698303222656, + "learning_rate": 3.9285205908608934e-07, + "logits/chosen": 0.5564339756965637, + "logits/rejected": 0.5101983547210693, + "logps/chosen": -90.3683853149414, + "logps/ref_chosen": -61.938209533691406, + "logps/ref_rejected": -72.21602630615234, + "logps/rejected": -120.01195526123047, + "loss": 0.9424, + "margin_dpo/margin_mean": 19.365753173828125, + "margin_dpo/margin_std": 25.310941696166992, + "step": 250 + }, + { + "KL/chosen_KL_mean": -30.75387954711914, + "KL/mean": -36.92249298095703, + "KL/rejected_KL_mean": -43.091102600097656, + "KL/std": 22.529541015625, + "epoch": 0.3794406651549509, + "fcm_dpo/beta": 0.04871663451194763, + "fcm_dpo/delta": -0.20225681364536285, + "fcm_dpo/margin": 12.337220191955566, + "fcm_dpo/q_t": 0.3839171528816223, + "grad_norm": 29.46482276916504, + "learning_rate": 3.9176496596569265e-07, + "logits/chosen": 0.5347196459770203, + "logits/rejected": 0.4927389621734619, + "logps/chosen": -97.61082458496094, + "logps/ref_chosen": -66.85694885253906, + "logps/ref_rejected": -84.83396911621094, + "logps/rejected": -127.92507934570312, + "loss": 1.1627, + "margin_dpo/margin_mean": 12.337220191955566, + "margin_dpo/margin_std": 24.319419860839844, + "step": 251 + }, + { + "KL/chosen_KL_mean": -29.73520278930664, + "KL/mean": -35.498878479003906, + "KL/rejected_KL_mean": -41.2625617980957, + "KL/std": 24.97641372680664, + "epoch": 0.38095238095238093, + "fcm_dpo/beta": 0.048638246953487396, + "fcm_dpo/delta": -0.16191521286964417, + "fcm_dpo/margin": 11.527353286743164, + "fcm_dpo/q_t": 0.38932251930236816, + "grad_norm": 31.459781646728516, + "learning_rate": 3.9067390737445254e-07, + "logits/chosen": 0.43054282665252686, + "logits/rejected": 0.37673041224479675, + "logps/chosen": -85.95913696289062, + "logps/ref_chosen": -56.22393035888672, + "logps/ref_rejected": -77.1136245727539, + "logps/rejected": -118.37619018554688, + "loss": 1.2555, + "margin_dpo/margin_mean": 11.527355194091797, + "margin_dpo/margin_std": 26.220972061157227, + "step": 252 + }, + { + "KL/chosen_KL_mean": -29.225950241088867, + "KL/mean": -36.115413665771484, + "KL/rejected_KL_mean": -43.0048713684082, + "KL/std": 22.882814407348633, + "epoch": 0.382464096749811, + "fcm_dpo/beta": 0.048262570053339005, + "fcm_dpo/delta": -0.2673663794994354, + "fcm_dpo/margin": 13.77892780303955, + "fcm_dpo/q_t": 0.371825635433197, + "grad_norm": 21.70552635192871, + "learning_rate": 3.8957891383162304e-07, + "logits/chosen": 0.5323972105979919, + "logits/rejected": 0.48898401856422424, + "logps/chosen": -81.43595886230469, + "logps/ref_chosen": -52.21001434326172, + "logps/ref_rejected": -58.75764846801758, + "logps/rejected": -101.76251983642578, + "loss": 1.0855, + "margin_dpo/margin_mean": 13.778926849365234, + "margin_dpo/margin_std": 22.40662384033203, + "step": 253 + }, + { + "KL/chosen_KL_mean": -30.63168716430664, + "KL/mean": -38.04819107055664, + "KL/rejected_KL_mean": -45.464691162109375, + "KL/std": 24.422290802001953, + "epoch": 0.3839758125472411, + "fcm_dpo/beta": 0.04799710959196091, + "fcm_dpo/delta": -0.314456582069397, + "fcm_dpo/margin": 14.833005905151367, + "fcm_dpo/q_t": 0.36847761273384094, + "grad_norm": 22.567625045776367, + "learning_rate": 3.884800159665276e-07, + "logits/chosen": 0.45149898529052734, + "logits/rejected": 0.39856594800949097, + "logps/chosen": -96.26801300048828, + "logps/ref_chosen": -65.63632202148438, + "logps/ref_rejected": -82.34425354003906, + "logps/rejected": -127.80894470214844, + "loss": 1.0615, + "margin_dpo/margin_mean": 14.83300495147705, + "margin_dpo/margin_std": 23.7720947265625, + "step": 254 + }, + { + "KL/chosen_KL_mean": -27.512184143066406, + "KL/mean": -37.109981536865234, + "KL/rejected_KL_mean": -46.70777130126953, + "KL/std": 24.98943519592285, + "epoch": 0.3854875283446712, + "fcm_dpo/beta": 0.04755797237157822, + "fcm_dpo/delta": -0.5179307460784912, + "fcm_dpo/margin": 19.195598602294922, + "fcm_dpo/q_t": 0.3373062014579773, + "grad_norm": 22.929805755615234, + "learning_rate": 3.873772445177015e-07, + "logits/chosen": 0.46128273010253906, + "logits/rejected": 0.4285234212875366, + "logps/chosen": -95.42327880859375, + "logps/ref_chosen": -67.91108703613281, + "logps/ref_rejected": -83.89114379882812, + "logps/rejected": -130.5989227294922, + "loss": 0.9669, + "margin_dpo/margin_mean": 19.195598602294922, + "margin_dpo/margin_std": 26.44408416748047, + "step": 255 + }, + { + "KL/chosen_KL_mean": -33.13542175292969, + "KL/mean": -41.397300720214844, + "KL/rejected_KL_mean": -49.65918731689453, + "KL/std": 22.247928619384766, + "epoch": 0.3869992441421013, + "fcm_dpo/beta": 0.047153837978839874, + "fcm_dpo/delta": -0.38249605894088745, + "fcm_dpo/margin": 16.523761749267578, + "fcm_dpo/q_t": 0.3605099618434906, + "grad_norm": 26.032716751098633, + "learning_rate": 3.862706303320329e-07, + "logits/chosen": 0.47902727127075195, + "logits/rejected": 0.41441649198532104, + "logps/chosen": -96.63540649414062, + "logps/ref_chosen": -63.49998474121094, + "logps/ref_rejected": -90.77104187011719, + "logps/rejected": -140.43023681640625, + "loss": 1.0679, + "margin_dpo/margin_mean": 16.523761749267578, + "margin_dpo/margin_std": 26.86455535888672, + "step": 256 + }, + { + "KL/chosen_KL_mean": -34.24513244628906, + "KL/mean": -44.3470458984375, + "KL/rejected_KL_mean": -54.448951721191406, + "KL/std": 26.300973892211914, + "epoch": 0.3885109599395314, + "fcm_dpo/beta": 0.04669519513845444, + "fcm_dpo/delta": -0.5487983226776123, + "fcm_dpo/margin": 20.203819274902344, + "fcm_dpo/q_t": 0.33863380551338196, + "grad_norm": 23.280733108520508, + "learning_rate": 3.851602043638994e-07, + "logits/chosen": 0.469465434551239, + "logits/rejected": 0.4032299220561981, + "logps/chosen": -104.84577941894531, + "logps/ref_chosen": -70.60064697265625, + "logps/ref_rejected": -108.58313751220703, + "logps/rejected": -163.03208923339844, + "loss": 0.9721, + "margin_dpo/margin_mean": 20.203821182250977, + "margin_dpo/margin_std": 28.9437198638916, + "step": 257 + }, + { + "KL/chosen_KL_mean": -33.56182861328125, + "KL/mean": -41.534019470214844, + "KL/rejected_KL_mean": -49.50621032714844, + "KL/std": 21.912500381469727, + "epoch": 0.3900226757369615, + "fcm_dpo/beta": 0.04636796936392784, + "fcm_dpo/delta": -0.34171295166015625, + "fcm_dpo/margin": 15.94438362121582, + "fcm_dpo/q_t": 0.34617486596107483, + "grad_norm": 24.333837509155273, + "learning_rate": 3.840459976743023e-07, + "logits/chosen": 0.5259693264961243, + "logits/rejected": 0.47170233726501465, + "logps/chosen": -92.81599426269531, + "logps/ref_chosen": -59.25416564941406, + "logps/ref_rejected": -85.58709716796875, + "logps/rejected": -135.0933074951172, + "loss": 0.9494, + "margin_dpo/margin_mean": 15.94438362121582, + "margin_dpo/margin_std": 19.48408317565918, + "step": 258 + }, + { + "KL/chosen_KL_mean": -27.79443359375, + "KL/mean": -39.75336456298828, + "KL/rejected_KL_mean": -51.7122917175293, + "KL/std": 25.747657775878906, + "epoch": 0.3915343915343915, + "fcm_dpo/beta": 0.04574854299426079, + "fcm_dpo/delta": -0.7024785280227661, + "fcm_dpo/margin": 23.917858123779297, + "fcm_dpo/q_t": 0.29727405309677124, + "grad_norm": 20.64354705810547, + "learning_rate": 3.8292804142999796e-07, + "logits/chosen": 0.4176882803440094, + "logits/rejected": 0.31484198570251465, + "logps/chosen": -93.22930908203125, + "logps/ref_chosen": -65.43487548828125, + "logps/ref_rejected": -95.41731262207031, + "logps/rejected": -147.12960815429688, + "loss": 0.8511, + "margin_dpo/margin_mean": 23.917861938476562, + "margin_dpo/margin_std": 26.349584579467773, + "step": 259 + }, + { + "KL/chosen_KL_mean": -29.042078018188477, + "KL/mean": -38.13416290283203, + "KL/rejected_KL_mean": -47.226234436035156, + "KL/std": 23.22982406616211, + "epoch": 0.3930461073318216, + "fcm_dpo/beta": 0.045296140015125275, + "fcm_dpo/delta": -0.4275883436203003, + "fcm_dpo/margin": 18.184162139892578, + "fcm_dpo/q_t": 0.3494893014431, + "grad_norm": 23.643529891967773, + "learning_rate": 3.818063669026256e-07, + "logits/chosen": 0.4728953242301941, + "logits/rejected": 0.38382914662361145, + "logps/chosen": -78.13166046142578, + "logps/ref_chosen": -49.08958435058594, + "logps/ref_rejected": -79.01708221435547, + "logps/rejected": -126.24331665039062, + "loss": 1.0368, + "margin_dpo/margin_mean": 18.184162139892578, + "margin_dpo/margin_std": 27.384994506835938, + "step": 260 + }, + { + "KL/chosen_KL_mean": -29.897396087646484, + "KL/mean": -37.54711151123047, + "KL/rejected_KL_mean": -45.19682312011719, + "KL/std": 23.129405975341797, + "epoch": 0.3945578231292517, + "fcm_dpo/beta": 0.045036204159259796, + "fcm_dpo/delta": -0.2910187244415283, + "fcm_dpo/margin": 15.299427032470703, + "fcm_dpo/q_t": 0.3705509901046753, + "grad_norm": 27.054941177368164, + "learning_rate": 3.806810054678331e-07, + "logits/chosen": 0.3471364974975586, + "logits/rejected": 0.3734210133552551, + "logps/chosen": -100.76979064941406, + "logps/ref_chosen": -70.87239074707031, + "logps/ref_rejected": -65.01522064208984, + "logps/rejected": -110.21204376220703, + "loss": 1.0598, + "margin_dpo/margin_mean": 15.299428939819336, + "margin_dpo/margin_std": 24.662092208862305, + "step": 261 + }, + { + "KL/chosen_KL_mean": -30.555986404418945, + "KL/mean": -39.38633346557617, + "KL/rejected_KL_mean": -48.21668243408203, + "KL/std": 22.795791625976562, + "epoch": 0.3960695389266818, + "fcm_dpo/beta": 0.044718965888023376, + "fcm_dpo/delta": -0.39276444911956787, + "fcm_dpo/margin": 17.660701751708984, + "fcm_dpo/q_t": 0.3487775921821594, + "grad_norm": 22.073932647705078, + "learning_rate": 3.7955198860439887e-07, + "logits/chosen": 0.5428476333618164, + "logits/rejected": 0.47747424244880676, + "logps/chosen": -98.4266128540039, + "logps/ref_chosen": -67.8706283569336, + "logps/ref_rejected": -88.7205810546875, + "logps/rejected": -136.937255859375, + "loss": 0.9769, + "margin_dpo/margin_mean": 17.660703659057617, + "margin_dpo/margin_std": 23.813310623168945, + "step": 262 + }, + { + "KL/chosen_KL_mean": -29.476200103759766, + "KL/mean": -36.817848205566406, + "KL/rejected_KL_mean": -44.15950393676758, + "KL/std": 23.008764266967773, + "epoch": 0.3975812547241119, + "fcm_dpo/beta": 0.04446381330490112, + "fcm_dpo/delta": -0.2544686794281006, + "fcm_dpo/margin": 14.683300018310547, + "fcm_dpo/q_t": 0.3705074191093445, + "grad_norm": 20.268625259399414, + "learning_rate": 3.784193478933516e-07, + "logits/chosen": 0.45436009764671326, + "logits/rejected": 0.34719711542129517, + "logps/chosen": -84.67078399658203, + "logps/ref_chosen": -55.194583892822266, + "logps/ref_rejected": -80.54048156738281, + "logps/rejected": -124.69998931884766, + "loss": 1.0803, + "margin_dpo/margin_mean": 14.683300018310547, + "margin_dpo/margin_std": 24.154205322265625, + "step": 263 + }, + { + "KL/chosen_KL_mean": -30.97838020324707, + "KL/mean": -39.79098892211914, + "KL/rejected_KL_mean": -48.603599548339844, + "KL/std": 25.080219268798828, + "epoch": 0.39909297052154197, + "fcm_dpo/beta": 0.0441419892013073, + "fcm_dpo/delta": -0.38093721866607666, + "fcm_dpo/margin": 17.62520980834961, + "fcm_dpo/q_t": 0.35710543394088745, + "grad_norm": 24.347915649414062, + "learning_rate": 3.7728311501708674e-07, + "logits/chosen": 0.4008549749851227, + "logits/rejected": 0.35340985655784607, + "logps/chosen": -114.14906311035156, + "logps/ref_chosen": -83.17068481445312, + "logps/ref_rejected": -88.33625793457031, + "logps/rejected": -136.93984985351562, + "loss": 1.0434, + "margin_dpo/margin_mean": 17.625211715698242, + "margin_dpo/margin_std": 27.51569366455078, + "step": 264 + }, + { + "KL/chosen_KL_mean": -29.328813552856445, + "KL/mean": -38.73906707763672, + "KL/rejected_KL_mean": -48.14931869506836, + "KL/std": 25.191619873046875, + "epoch": 0.40060468631897206, + "fcm_dpo/beta": 0.04379952326416969, + "fcm_dpo/delta": -0.4276652932167053, + "fcm_dpo/margin": 18.82050323486328, + "fcm_dpo/q_t": 0.3490108847618103, + "grad_norm": 23.407211303710938, + "learning_rate": 3.7614332175848027e-07, + "logits/chosen": 0.5712328553199768, + "logits/rejected": 0.5019096732139587, + "logps/chosen": -80.99166870117188, + "logps/ref_chosen": -51.66284942626953, + "logps/ref_rejected": -67.1720962524414, + "logps/rejected": -115.3214111328125, + "loss": 1.0741, + "margin_dpo/margin_mean": 18.82050132751465, + "margin_dpo/margin_std": 30.062911987304688, + "step": 265 + }, + { + "KL/chosen_KL_mean": -27.56685447692871, + "KL/mean": -36.56960678100586, + "KL/rejected_KL_mean": -45.57236099243164, + "KL/std": 25.584293365478516, + "epoch": 0.4021164021164021, + "fcm_dpo/beta": 0.04341866821050644, + "fcm_dpo/delta": -0.38486039638519287, + "fcm_dpo/margin": 18.005504608154297, + "fcm_dpo/q_t": 0.3500235378742218, + "grad_norm": 21.611257553100586, + "learning_rate": 3.75e-07, + "logits/chosen": 0.48965033888816833, + "logits/rejected": 0.4134351313114166, + "logps/chosen": -85.01734924316406, + "logps/ref_chosen": -57.45049285888672, + "logps/ref_rejected": -77.60826110839844, + "logps/rejected": -123.18061828613281, + "loss": 0.9928, + "margin_dpo/margin_mean": 18.005504608154297, + "margin_dpo/margin_std": 25.501304626464844, + "step": 266 + }, + { + "KL/chosen_KL_mean": -23.97246551513672, + "KL/mean": -30.96150779724121, + "KL/rejected_KL_mean": -37.95055389404297, + "KL/std": 22.680313110351562, + "epoch": 0.4036281179138322, + "fcm_dpo/beta": 0.04327762871980667, + "fcm_dpo/delta": -0.2063625603914261, + "fcm_dpo/margin": 13.978081703186035, + "fcm_dpo/q_t": 0.38236600160598755, + "grad_norm": 21.35399627685547, + "learning_rate": 3.738531817228131e-07, + "logits/chosen": 0.49754825234413147, + "logits/rejected": 0.47748124599456787, + "logps/chosen": -79.0078125, + "logps/ref_chosen": -55.03535079956055, + "logps/ref_rejected": -66.0953369140625, + "logps/rejected": -104.04588317871094, + "loss": 1.1529, + "margin_dpo/margin_mean": 13.978081703186035, + "margin_dpo/margin_std": 25.985679626464844, + "step": 267 + }, + { + "KL/chosen_KL_mean": -24.13412857055664, + "KL/mean": -32.38087463378906, + "KL/rejected_KL_mean": -40.62763214111328, + "KL/std": 24.921459197998047, + "epoch": 0.4051398337112623, + "fcm_dpo/beta": 0.04298625886440277, + "fcm_dpo/delta": -0.31109189987182617, + "fcm_dpo/margin": 16.49349594116211, + "fcm_dpo/q_t": 0.3614717423915863, + "grad_norm": 17.274066925048828, + "learning_rate": 3.7270289900589204e-07, + "logits/chosen": 0.3792925477027893, + "logits/rejected": 0.36041849851608276, + "logps/chosen": -89.20587158203125, + "logps/ref_chosen": -65.07174682617188, + "logps/ref_rejected": -71.42485809326172, + "logps/rejected": -112.052490234375, + "loss": 0.9883, + "margin_dpo/margin_mean": 16.493497848510742, + "margin_dpo/margin_std": 22.391368865966797, + "step": 268 + }, + { + "KL/chosen_KL_mean": -26.810436248779297, + "KL/mean": -36.081504821777344, + "KL/rejected_KL_mean": -45.35257339477539, + "KL/std": 25.351978302001953, + "epoch": 0.40665154950869237, + "fcm_dpo/beta": 0.04269479587674141, + "fcm_dpo/delta": -0.39459335803985596, + "fcm_dpo/margin": 18.542144775390625, + "fcm_dpo/q_t": 0.3451964557170868, + "grad_norm": 17.133892059326172, + "learning_rate": 3.7154918402511714e-07, + "logits/chosen": 0.5849369764328003, + "logits/rejected": 0.533880889415741, + "logps/chosen": -93.94664001464844, + "logps/ref_chosen": -67.1362075805664, + "logps/ref_rejected": -82.55778503417969, + "logps/rejected": -127.91036224365234, + "loss": 0.9608, + "margin_dpo/margin_mean": 18.542144775390625, + "margin_dpo/margin_std": 23.505943298339844, + "step": 269 + }, + { + "KL/chosen_KL_mean": -26.583717346191406, + "KL/mean": -34.255470275878906, + "KL/rejected_KL_mean": -41.927215576171875, + "KL/std": 24.510047912597656, + "epoch": 0.40816326530612246, + "fcm_dpo/beta": 0.042380329221487045, + "fcm_dpo/delta": -0.25204920768737793, + "fcm_dpo/margin": 15.343496322631836, + "fcm_dpo/q_t": 0.36615848541259766, + "grad_norm": 21.264305114746094, + "learning_rate": 3.7039206905237656e-07, + "logits/chosen": 0.5199756622314453, + "logits/rejected": 0.43893975019454956, + "logps/chosen": -93.27241516113281, + "logps/ref_chosen": -66.6886978149414, + "logps/ref_rejected": -85.16129302978516, + "logps/rejected": -127.08850860595703, + "loss": 1.0463, + "margin_dpo/margin_mean": 15.343497276306152, + "margin_dpo/margin_std": 23.583026885986328, + "step": 270 + }, + { + "KL/chosen_KL_mean": -25.026885986328125, + "KL/mean": -31.79044532775879, + "KL/rejected_KL_mean": -38.553993225097656, + "KL/std": 25.43020248413086, + "epoch": 0.40967498110355255, + "fcm_dpo/beta": 0.042209602892398834, + "fcm_dpo/delta": -0.17214374244213104, + "fcm_dpo/margin": 13.527109146118164, + "fcm_dpo/q_t": 0.4038546085357666, + "grad_norm": 22.149560928344727, + "learning_rate": 3.692315864546635e-07, + "logits/chosen": 0.5008272528648376, + "logits/rejected": 0.4377139210700989, + "logps/chosen": -97.43443298339844, + "logps/ref_chosen": -72.40754699707031, + "logps/ref_rejected": -92.06311798095703, + "logps/rejected": -130.6171112060547, + "loss": 1.1983, + "margin_dpo/margin_mean": 13.527109146118164, + "margin_dpo/margin_std": 29.24249267578125, + "step": 271 + }, + { + "KL/chosen_KL_mean": -21.254152297973633, + "KL/mean": -32.72765350341797, + "KL/rejected_KL_mean": -44.20115280151367, + "KL/std": 23.741710662841797, + "epoch": 0.41118669690098264, + "fcm_dpo/beta": 0.04187573492527008, + "fcm_dpo/delta": -0.5660500526428223, + "fcm_dpo/margin": 22.94700050354004, + "fcm_dpo/q_t": 0.3139285743236542, + "grad_norm": 19.577409744262695, + "learning_rate": 3.6806776869317067e-07, + "logits/chosen": 0.49642032384872437, + "logits/rejected": 0.5086947679519653, + "logps/chosen": -87.85555267333984, + "logps/ref_chosen": -66.60140228271484, + "logps/ref_rejected": -67.74340057373047, + "logps/rejected": -111.94454956054688, + "loss": 0.83, + "margin_dpo/margin_mean": 22.94700050354004, + "margin_dpo/margin_std": 23.4195556640625, + "step": 272 + }, + { + "KL/chosen_KL_mean": -28.709854125976562, + "KL/mean": -37.45901107788086, + "KL/rejected_KL_mean": -46.208168029785156, + "KL/std": 25.393630981445312, + "epoch": 0.4126984126984127, + "fcm_dpo/beta": 0.04153010621666908, + "fcm_dpo/delta": -0.32902804017066956, + "fcm_dpo/margin": 17.498313903808594, + "fcm_dpo/q_t": 0.3637559711933136, + "grad_norm": 22.20293617248535, + "learning_rate": 3.669006483223828e-07, + "logits/chosen": 0.5220750570297241, + "logits/rejected": 0.45149755477905273, + "logps/chosen": -86.06472778320312, + "logps/ref_chosen": -57.35487747192383, + "logps/ref_rejected": -84.17168426513672, + "logps/rejected": -130.37985229492188, + "loss": 1.0746, + "margin_dpo/margin_mean": 17.498313903808594, + "margin_dpo/margin_std": 29.040061950683594, + "step": 273 + }, + { + "KL/chosen_KL_mean": -26.842775344848633, + "KL/mean": -35.87696838378906, + "KL/rejected_KL_mean": -44.91115951538086, + "KL/std": 24.45706558227539, + "epoch": 0.41421012849584277, + "fcm_dpo/beta": 0.041252560913562775, + "fcm_dpo/delta": -0.34785181283950806, + "fcm_dpo/margin": 18.06838035583496, + "fcm_dpo/q_t": 0.3558685779571533, + "grad_norm": 18.208568572998047, + "learning_rate": 3.657302579891656e-07, + "logits/chosen": 0.3852129280567169, + "logits/rejected": 0.3623676896095276, + "logps/chosen": -86.48426818847656, + "logps/ref_chosen": -59.64149475097656, + "logps/ref_rejected": -68.29348754882812, + "logps/rejected": -113.20464324951172, + "loss": 1.021, + "margin_dpo/margin_mean": 18.06838035583496, + "margin_dpo/margin_std": 26.578868865966797, + "step": 274 + }, + { + "KL/chosen_KL_mean": -26.34328842163086, + "KL/mean": -36.42488479614258, + "KL/rejected_KL_mean": -46.50648498535156, + "KL/std": 25.752819061279297, + "epoch": 0.41572184429327286, + "fcm_dpo/beta": 0.04092035070061684, + "fcm_dpo/delta": -0.42846059799194336, + "fcm_dpo/margin": 20.163192749023438, + "fcm_dpo/q_t": 0.3388616740703583, + "grad_norm": 18.95833969116211, + "learning_rate": 3.645566304318526e-07, + "logits/chosen": 0.46770888566970825, + "logits/rejected": 0.3843296766281128, + "logps/chosen": -79.60993194580078, + "logps/ref_chosen": -53.26664352416992, + "logps/ref_rejected": -73.84062194824219, + "logps/rejected": -120.34710693359375, + "loss": 0.9533, + "margin_dpo/margin_mean": 20.163192749023438, + "margin_dpo/margin_std": 26.204391479492188, + "step": 275 + }, + { + "KL/chosen_KL_mean": -24.395612716674805, + "KL/mean": -33.88249206542969, + "KL/rejected_KL_mean": -43.369361877441406, + "KL/std": 23.688095092773438, + "epoch": 0.41723356009070295, + "fcm_dpo/beta": 0.040573496371507645, + "fcm_dpo/delta": -0.37282446026802063, + "fcm_dpo/margin": 18.973751068115234, + "fcm_dpo/q_t": 0.34385862946510315, + "grad_norm": 18.067203521728516, + "learning_rate": 3.633797984793294e-07, + "logits/chosen": 0.4242190718650818, + "logits/rejected": 0.38918283581733704, + "logps/chosen": -77.41641235351562, + "logps/ref_chosen": -53.02079772949219, + "logps/ref_rejected": -61.56678771972656, + "logps/rejected": -104.9361572265625, + "loss": 0.9339, + "margin_dpo/margin_mean": 18.973751068115234, + "margin_dpo/margin_std": 22.783233642578125, + "step": 276 + }, + { + "KL/chosen_KL_mean": -28.17159080505371, + "KL/mean": -34.041229248046875, + "KL/rejected_KL_mean": -39.910858154296875, + "KL/std": 23.754390716552734, + "epoch": 0.41874527588813304, + "fcm_dpo/beta": 0.0404670313000679, + "fcm_dpo/delta": -0.07540662586688995, + "fcm_dpo/margin": 11.73927116394043, + "fcm_dpo/q_t": 0.40724635124206543, + "grad_norm": 23.81082534790039, + "learning_rate": 3.6219979505011555e-07, + "logits/chosen": 0.5104623436927795, + "logits/rejected": 0.5346454381942749, + "logps/chosen": -99.60458374023438, + "logps/ref_chosen": -71.43299102783203, + "logps/ref_rejected": -67.65852355957031, + "logps/rejected": -107.56938171386719, + "loss": 1.171, + "margin_dpo/margin_mean": 11.73927116394043, + "margin_dpo/margin_std": 24.22088623046875, + "step": 277 + }, + { + "KL/chosen_KL_mean": -29.232479095458984, + "KL/mean": -38.33482360839844, + "KL/rejected_KL_mean": -47.437164306640625, + "KL/std": 27.14554214477539, + "epoch": 0.42025699168556313, + "fcm_dpo/beta": 0.04029684513807297, + "fcm_dpo/delta": -0.3359553813934326, + "fcm_dpo/margin": 18.204681396484375, + "fcm_dpo/q_t": 0.35321781039237976, + "grad_norm": 23.709562301635742, + "learning_rate": 3.6101665315144353e-07, + "logits/chosen": 0.4153169095516205, + "logits/rejected": 0.3612356185913086, + "logps/chosen": -96.34324645996094, + "logps/ref_chosen": -67.11076354980469, + "logps/ref_rejected": -88.74851989746094, + "logps/rejected": -136.18568420410156, + "loss": 1.0209, + "margin_dpo/margin_mean": 18.204681396484375, + "margin_dpo/margin_std": 26.00763511657715, + "step": 278 + }, + { + "KL/chosen_KL_mean": -21.19234848022461, + "KL/mean": -33.03874969482422, + "KL/rejected_KL_mean": -44.88514709472656, + "KL/std": 25.397165298461914, + "epoch": 0.4217687074829932, + "fcm_dpo/beta": 0.03986917808651924, + "fcm_dpo/delta": -0.5495724678039551, + "fcm_dpo/margin": 23.692798614501953, + "fcm_dpo/q_t": 0.3151549696922302, + "grad_norm": 19.516904830932617, + "learning_rate": 3.5983040587833563e-07, + "logits/chosen": 0.4501017928123474, + "logits/rejected": 0.4077523350715637, + "logps/chosen": -75.68983459472656, + "logps/ref_chosen": -54.49748611450195, + "logps/ref_rejected": -70.42373657226562, + "logps/rejected": -115.30888366699219, + "loss": 0.8413, + "margin_dpo/margin_mean": 23.692798614501953, + "margin_dpo/margin_std": 24.541980743408203, + "step": 279 + }, + { + "KL/chosen_KL_mean": -23.100849151611328, + "KL/mean": -35.334835052490234, + "KL/rejected_KL_mean": -47.568824768066406, + "KL/std": 25.5330810546875, + "epoch": 0.42328042328042326, + "fcm_dpo/beta": 0.03940429538488388, + "fcm_dpo/delta": -0.5695863962173462, + "fcm_dpo/margin": 24.46797752380371, + "fcm_dpo/q_t": 0.3104165494441986, + "grad_norm": 18.322284698486328, + "learning_rate": 3.586410864126781e-07, + "logits/chosen": 0.5149914026260376, + "logits/rejected": 0.4684107005596161, + "logps/chosen": -83.53366088867188, + "logps/ref_chosen": -60.43281173706055, + "logps/ref_rejected": -78.39051818847656, + "logps/rejected": -125.95934295654297, + "loss": 0.8024, + "margin_dpo/margin_mean": 24.467975616455078, + "margin_dpo/margin_std": 23.696447372436523, + "step": 280 + }, + { + "KL/chosen_KL_mean": -26.12680435180664, + "KL/mean": -37.30401611328125, + "KL/rejected_KL_mean": -48.481231689453125, + "KL/std": 26.863807678222656, + "epoch": 0.42479213907785335, + "fcm_dpo/beta": 0.03900133818387985, + "fcm_dpo/delta": -0.4761030673980713, + "fcm_dpo/margin": 22.35442543029785, + "fcm_dpo/q_t": 0.32904618978500366, + "grad_norm": 16.90000343322754, + "learning_rate": 3.574487280222929e-07, + "logits/chosen": 0.4731192886829376, + "logits/rejected": 0.4898528754711151, + "logps/chosen": -86.40889739990234, + "logps/ref_chosen": -60.2820930480957, + "logps/ref_rejected": -62.04009246826172, + "logps/rejected": -110.52132415771484, + "loss": 0.9089, + "margin_dpo/margin_mean": 22.35442543029785, + "margin_dpo/margin_std": 26.479328155517578, + "step": 281 + }, + { + "KL/chosen_KL_mean": -29.51664161682129, + "KL/mean": -40.19871520996094, + "KL/rejected_KL_mean": -50.88078689575195, + "KL/std": 26.441116333007812, + "epoch": 0.42630385487528344, + "fcm_dpo/beta": 0.03872400522232056, + "fcm_dpo/delta": -0.43067678809165955, + "fcm_dpo/margin": 21.3641414642334, + "fcm_dpo/q_t": 0.3469800651073456, + "grad_norm": 21.590852737426758, + "learning_rate": 3.562533640600075e-07, + "logits/chosen": 0.4241343140602112, + "logits/rejected": 0.37093585729599, + "logps/chosen": -90.14056396484375, + "logps/ref_chosen": -60.623924255371094, + "logps/ref_rejected": -68.67400360107422, + "logps/rejected": -119.55479431152344, + "loss": 0.976, + "margin_dpo/margin_mean": 21.36414337158203, + "margin_dpo/margin_std": 28.510835647583008, + "step": 282 + }, + { + "KL/chosen_KL_mean": -31.545570373535156, + "KL/mean": -40.97801971435547, + "KL/rejected_KL_mean": -50.41047668457031, + "KL/std": 27.320905685424805, + "epoch": 0.42781557067271353, + "fcm_dpo/beta": 0.03842076286673546, + "fcm_dpo/delta": -0.3270444869995117, + "fcm_dpo/margin": 18.864906311035156, + "fcm_dpo/q_t": 0.36605554819107056, + "grad_norm": 22.739614486694336, + "learning_rate": 3.550550279627215e-07, + "logits/chosen": 0.4676963686943054, + "logits/rejected": 0.3664717972278595, + "logps/chosen": -99.19331359863281, + "logps/ref_chosen": -67.64775085449219, + "logps/ref_rejected": -99.96835327148438, + "logps/rejected": -150.3788299560547, + "loss": 1.0565, + "margin_dpo/margin_mean": 18.86490821838379, + "margin_dpo/margin_std": 29.8179931640625, + "step": 283 + }, + { + "KL/chosen_KL_mean": -30.34023666381836, + "KL/mean": -40.260833740234375, + "KL/rejected_KL_mean": -50.181434631347656, + "KL/std": 29.524093627929688, + "epoch": 0.4293272864701436, + "fcm_dpo/beta": 0.03809501603245735, + "fcm_dpo/delta": -0.3589645326137543, + "fcm_dpo/margin": 19.841190338134766, + "fcm_dpo/q_t": 0.3523881137371063, + "grad_norm": 20.68400001525879, + "learning_rate": 3.5385375325047163e-07, + "logits/chosen": 0.5279954671859741, + "logits/rejected": 0.4627271294593811, + "logps/chosen": -87.30766296386719, + "logps/ref_chosen": -56.96742630004883, + "logps/ref_rejected": -86.36236572265625, + "logps/rejected": -136.54380798339844, + "loss": 0.9769, + "margin_dpo/margin_mean": 19.841190338134766, + "margin_dpo/margin_std": 27.096464157104492, + "step": 284 + }, + { + "KL/chosen_KL_mean": -35.98102569580078, + "KL/mean": -45.11893844604492, + "KL/rejected_KL_mean": -54.25685119628906, + "KL/std": 26.50765609741211, + "epoch": 0.4308390022675737, + "fcm_dpo/beta": 0.0379050076007843, + "fcm_dpo/delta": -0.2946973741054535, + "fcm_dpo/margin": 18.27582359313965, + "fcm_dpo/q_t": 0.36850297451019287, + "grad_norm": 21.635417938232422, + "learning_rate": 3.5264957352549375e-07, + "logits/chosen": 0.5534378290176392, + "logits/rejected": 0.523492693901062, + "logps/chosen": -107.63713073730469, + "logps/ref_chosen": -71.65611267089844, + "logps/ref_rejected": -81.63829803466797, + "logps/rejected": -135.8951416015625, + "loss": 1.0347, + "margin_dpo/margin_mean": 18.27582550048828, + "margin_dpo/margin_std": 27.676040649414062, + "step": 285 + }, + { + "KL/chosen_KL_mean": -35.323272705078125, + "KL/mean": -49.52436065673828, + "KL/rejected_KL_mean": -63.7254524230957, + "KL/std": 29.0585880279541, + "epoch": 0.4323507180650038, + "fcm_dpo/beta": 0.03745154291391373, + "fcm_dpo/delta": -0.6708974242210388, + "fcm_dpo/margin": 28.402177810668945, + "fcm_dpo/q_t": 0.29910317063331604, + "grad_norm": 18.906368255615234, + "learning_rate": 3.514425224712835e-07, + "logits/chosen": 0.4541959762573242, + "logits/rejected": 0.35684093832969666, + "logps/chosen": -96.40279388427734, + "logps/ref_chosen": -61.07952117919922, + "logps/ref_rejected": -91.28128051757812, + "logps/rejected": -155.00672912597656, + "loss": 0.8133, + "margin_dpo/margin_mean": 28.402177810668945, + "margin_dpo/margin_std": 29.681446075439453, + "step": 286 + }, + { + "KL/chosen_KL_mean": -29.999765396118164, + "KL/mean": -44.197662353515625, + "KL/rejected_KL_mean": -58.39556884765625, + "KL/std": 31.966909408569336, + "epoch": 0.43386243386243384, + "fcm_dpo/beta": 0.036974288523197174, + "fcm_dpo/delta": -0.6566334962844849, + "fcm_dpo/margin": 28.395801544189453, + "fcm_dpo/q_t": 0.30857691168785095, + "grad_norm": 18.613101959228516, + "learning_rate": 3.502326338516534e-07, + "logits/chosen": 0.5451105833053589, + "logits/rejected": 0.5025367140769958, + "logps/chosen": -76.03555297851562, + "logps/ref_chosen": -46.035789489746094, + "logps/ref_rejected": -59.95293426513672, + "logps/rejected": -118.34850311279297, + "loss": 0.8407, + "margin_dpo/margin_mean": 28.395801544189453, + "margin_dpo/margin_std": 31.618913650512695, + "step": 287 + }, + { + "KL/chosen_KL_mean": -39.62640380859375, + "KL/mean": -48.881690979003906, + "KL/rejected_KL_mean": -58.136985778808594, + "KL/std": 30.076122283935547, + "epoch": 0.43537414965986393, + "fcm_dpo/beta": 0.03669974207878113, + "fcm_dpo/delta": -0.28116920590400696, + "fcm_dpo/margin": 18.510595321655273, + "fcm_dpo/q_t": 0.37004995346069336, + "grad_norm": 23.904186248779297, + "learning_rate": 3.490199415097892e-07, + "logits/chosen": 0.3931962549686432, + "logits/rejected": 0.33491191267967224, + "logps/chosen": -105.01725006103516, + "logps/ref_chosen": -65.3908462524414, + "logps/ref_rejected": -88.53607940673828, + "logps/rejected": -146.67306518554688, + "loss": 1.0884, + "margin_dpo/margin_mean": 18.510595321655273, + "margin_dpo/margin_std": 31.303096771240234, + "step": 288 + }, + { + "KL/chosen_KL_mean": -40.47001266479492, + "KL/mean": -50.500709533691406, + "KL/rejected_KL_mean": -60.53139877319336, + "KL/std": 30.149703979492188, + "epoch": 0.436885865457294, + "fcm_dpo/beta": 0.036467455327510834, + "fcm_dpo/delta": -0.33392614126205444, + "fcm_dpo/margin": 20.061397552490234, + "fcm_dpo/q_t": 0.3704224228858948, + "grad_norm": 20.787437438964844, + "learning_rate": 3.4780447936730247e-07, + "logits/chosen": 0.6029109954833984, + "logits/rejected": 0.5617333650588989, + "logps/chosen": -95.06364440917969, + "logps/ref_chosen": -54.5936279296875, + "logps/ref_rejected": -67.20855712890625, + "logps/rejected": -127.73995208740234, + "loss": 1.087, + "margin_dpo/margin_mean": 20.061397552490234, + "margin_dpo/margin_std": 33.538150787353516, + "step": 289 + }, + { + "KL/chosen_KL_mean": -45.79582977294922, + "KL/mean": -58.52228927612305, + "KL/rejected_KL_mean": -71.24874877929688, + "KL/std": 30.670194625854492, + "epoch": 0.4383975812547241, + "fcm_dpo/beta": 0.03608309477567673, + "fcm_dpo/delta": -0.5236524343490601, + "fcm_dpo/margin": 25.452919006347656, + "fcm_dpo/q_t": 0.3284386992454529, + "grad_norm": 25.504919052124023, + "learning_rate": 3.465862814232821e-07, + "logits/chosen": 0.6134675741195679, + "logits/rejected": 0.5394095182418823, + "logps/chosen": -107.18040466308594, + "logps/ref_chosen": -61.38457489013672, + "logps/ref_rejected": -91.92778015136719, + "logps/rejected": -163.17652893066406, + "loss": 0.953, + "margin_dpo/margin_mean": 25.452919006347656, + "margin_dpo/margin_std": 33.5854377746582, + "step": 290 + }, + { + "KL/chosen_KL_mean": -43.414634704589844, + "KL/mean": -56.460289001464844, + "KL/rejected_KL_mean": -69.50595092773438, + "KL/std": 30.899322509765625, + "epoch": 0.4399092970521542, + "fcm_dpo/beta": 0.035775572061538696, + "fcm_dpo/delta": -0.5381011366844177, + "fcm_dpo/margin": 26.09131622314453, + "fcm_dpo/q_t": 0.3339788615703583, + "grad_norm": 22.091732025146484, + "learning_rate": 3.4536538175334343e-07, + "logits/chosen": 0.6651061773300171, + "logits/rejected": 0.5917015075683594, + "logps/chosen": -94.27766418457031, + "logps/ref_chosen": -50.863037109375, + "logps/ref_rejected": -82.20868682861328, + "logps/rejected": -151.71463012695312, + "loss": 0.955, + "margin_dpo/margin_mean": 26.09131622314453, + "margin_dpo/margin_std": 34.50392150878906, + "step": 291 + }, + { + "KL/chosen_KL_mean": -42.59569549560547, + "KL/mean": -53.231895446777344, + "KL/rejected_KL_mean": -63.86809539794922, + "KL/std": 31.110265731811523, + "epoch": 0.4414210128495843, + "fcm_dpo/beta": 0.03544076532125473, + "fcm_dpo/delta": -0.35655850172042847, + "fcm_dpo/margin": 21.272396087646484, + "fcm_dpo/q_t": 0.355457067489624, + "grad_norm": 22.543594360351562, + "learning_rate": 3.4414181450867465e-07, + "logits/chosen": 0.5674319267272949, + "logits/rejected": 0.5143272876739502, + "logps/chosen": -106.944580078125, + "logps/ref_chosen": -64.34888458251953, + "logps/ref_rejected": -72.86434173583984, + "logps/rejected": -136.73243713378906, + "loss": 1.0462, + "margin_dpo/margin_mean": 21.272396087646484, + "margin_dpo/margin_std": 33.215049743652344, + "step": 292 + }, + { + "KL/chosen_KL_mean": -45.501914978027344, + "KL/mean": -60.231849670410156, + "KL/rejected_KL_mean": -74.96177673339844, + "KL/std": 32.89813232421875, + "epoch": 0.4429327286470144, + "fcm_dpo/beta": 0.0350659117102623, + "fcm_dpo/delta": -0.6392489671707153, + "fcm_dpo/margin": 29.459863662719727, + "fcm_dpo/q_t": 0.3127155005931854, + "grad_norm": 17.226879119873047, + "learning_rate": 3.4291561391508185e-07, + "logits/chosen": 0.6422121524810791, + "logits/rejected": 0.5492717027664185, + "logps/chosen": -100.37138366699219, + "logps/ref_chosen": -54.869468688964844, + "logps/ref_rejected": -81.858642578125, + "logps/rejected": -156.8204345703125, + "loss": 0.9196, + "margin_dpo/margin_mean": 29.459863662719727, + "margin_dpo/margin_std": 36.631168365478516, + "step": 293 + }, + { + "KL/chosen_KL_mean": -44.19648361206055, + "KL/mean": -55.805946350097656, + "KL/rejected_KL_mean": -67.4154052734375, + "KL/std": 34.420440673828125, + "epoch": 0.4444444444444444, + "fcm_dpo/beta": 0.03468414396047592, + "fcm_dpo/delta": -0.4090438485145569, + "fcm_dpo/margin": 23.218923568725586, + "fcm_dpo/q_t": 0.35191962122917175, + "grad_norm": 19.047632217407227, + "learning_rate": 3.4168681427203153e-07, + "logits/chosen": 0.5739268064498901, + "logits/rejected": 0.5258492231369019, + "logps/chosen": -100.86738586425781, + "logps/ref_chosen": -56.670902252197266, + "logps/ref_rejected": -70.32819366455078, + "logps/rejected": -137.74359130859375, + "loss": 0.965, + "margin_dpo/margin_mean": 23.218923568725586, + "margin_dpo/margin_std": 31.80727767944336, + "step": 294 + }, + { + "KL/chosen_KL_mean": -49.95307159423828, + "KL/mean": -60.020530700683594, + "KL/rejected_KL_mean": -70.0879898071289, + "KL/std": 33.080787658691406, + "epoch": 0.4459561602418745, + "fcm_dpo/beta": 0.0344795361161232, + "fcm_dpo/delta": -0.29635918140411377, + "fcm_dpo/margin": 20.13490867614746, + "fcm_dpo/q_t": 0.37185660004615784, + "grad_norm": 24.184593200683594, + "learning_rate": 3.4045544995169125e-07, + "logits/chosen": 0.5651549696922302, + "logits/rejected": 0.45998790860176086, + "logps/chosen": -100.35395812988281, + "logps/ref_chosen": -50.40088653564453, + "logps/ref_rejected": -83.43521881103516, + "logps/rejected": -153.52320861816406, + "loss": 1.0582, + "margin_dpo/margin_mean": 20.134910583496094, + "margin_dpo/margin_std": 32.56691360473633, + "step": 295 + }, + { + "KL/chosen_KL_mean": -49.076019287109375, + "KL/mean": -61.38549041748047, + "KL/rejected_KL_mean": -73.69496154785156, + "KL/std": 33.86975860595703, + "epoch": 0.4474678760393046, + "fcm_dpo/beta": 0.03418150916695595, + "fcm_dpo/delta": -0.4456692039966583, + "fcm_dpo/margin": 24.618946075439453, + "fcm_dpo/q_t": 0.34331923723220825, + "grad_norm": 21.796897888183594, + "learning_rate": 3.392215553979679e-07, + "logits/chosen": 0.49833643436431885, + "logits/rejected": 0.45150434970855713, + "logps/chosen": -118.22636413574219, + "logps/ref_chosen": -69.15034484863281, + "logps/ref_rejected": -89.60166931152344, + "logps/rejected": -163.296630859375, + "loss": 0.9907, + "margin_dpo/margin_mean": 24.618946075439453, + "margin_dpo/margin_std": 34.83667755126953, + "step": 296 + }, + { + "KL/chosen_KL_mean": -51.80224609375, + "KL/mean": -65.018798828125, + "KL/rejected_KL_mean": -78.2353515625, + "KL/std": 30.489700317382812, + "epoch": 0.4489795918367347, + "fcm_dpo/beta": 0.033906228840351105, + "fcm_dpo/delta": -0.5004413723945618, + "fcm_dpo/margin": 26.43310546875, + "fcm_dpo/q_t": 0.3267863094806671, + "grad_norm": 19.539833068847656, + "learning_rate": 3.3798516512554485e-07, + "logits/chosen": 0.5151644945144653, + "logits/rejected": 0.45623141527175903, + "logps/chosen": -109.81855773925781, + "logps/ref_chosen": -58.01630401611328, + "logps/ref_rejected": -69.95780944824219, + "logps/rejected": -148.1931610107422, + "loss": 0.8924, + "margin_dpo/margin_mean": 26.43310546875, + "margin_dpo/margin_std": 30.026384353637695, + "step": 297 + }, + { + "KL/chosen_KL_mean": -51.99375915527344, + "KL/mean": -62.09900665283203, + "KL/rejected_KL_mean": -72.20426940917969, + "KL/std": 32.644649505615234, + "epoch": 0.4504913076341648, + "fcm_dpo/beta": 0.033642496913671494, + "fcm_dpo/delta": -0.28188586235046387, + "fcm_dpo/margin": 20.21050262451172, + "fcm_dpo/q_t": 0.3743436932563782, + "grad_norm": 22.63440704345703, + "learning_rate": 3.367463137189156e-07, + "logits/chosen": 0.6429187059402466, + "logits/rejected": 0.5825469493865967, + "logps/chosen": -108.16307067871094, + "logps/ref_chosen": -56.1693115234375, + "logps/ref_rejected": -68.55052185058594, + "logps/rejected": -140.75479125976562, + "loss": 1.1049, + "margin_dpo/margin_mean": 20.21050262451172, + "margin_dpo/margin_std": 35.36993408203125, + "step": 298 + }, + { + "KL/chosen_KL_mean": -49.722076416015625, + "KL/mean": -59.63947296142578, + "KL/rejected_KL_mean": -69.55687713623047, + "KL/std": 31.16931915283203, + "epoch": 0.4520030234315949, + "fcm_dpo/beta": 0.03344232589006424, + "fcm_dpo/delta": -0.26536402106285095, + "fcm_dpo/margin": 19.834793090820312, + "fcm_dpo/q_t": 0.377312034368515, + "grad_norm": 22.197856903076172, + "learning_rate": 3.355050358314172e-07, + "logits/chosen": 0.4444432258605957, + "logits/rejected": 0.4144965410232544, + "logps/chosen": -112.03987884521484, + "logps/ref_chosen": -62.31780242919922, + "logps/ref_rejected": -72.60028839111328, + "logps/rejected": -142.15716552734375, + "loss": 1.1208, + "margin_dpo/margin_mean": 19.834793090820312, + "margin_dpo/margin_std": 35.16393280029297, + "step": 299 + }, + { + "KL/chosen_KL_mean": -52.10127258300781, + "KL/mean": -63.23472595214844, + "KL/rejected_KL_mean": -74.36817932128906, + "KL/std": 31.207637786865234, + "epoch": 0.45351473922902497, + "fcm_dpo/beta": 0.03325977548956871, + "fcm_dpo/delta": -0.34305012226104736, + "fcm_dpo/margin": 22.266916275024414, + "fcm_dpo/q_t": 0.35531795024871826, + "grad_norm": 20.453025817871094, + "learning_rate": 3.3426136618426043e-07, + "logits/chosen": 0.5766974687576294, + "logits/rejected": 0.5067134499549866, + "logps/chosen": -112.48284912109375, + "logps/ref_chosen": -60.38157653808594, + "logps/ref_rejected": -75.45442199707031, + "logps/rejected": -149.82260131835938, + "loss": 1.0245, + "margin_dpo/margin_mean": 22.266916275024414, + "margin_dpo/margin_std": 32.75130844116211, + "step": 300 + }, + { + "KL/chosen_KL_mean": -49.72856140136719, + "KL/mean": -59.65312957763672, + "KL/rejected_KL_mean": -69.57768249511719, + "KL/std": 31.22908592224121, + "epoch": 0.455026455026455, + "fcm_dpo/beta": 0.033046964555978775, + "fcm_dpo/delta": -0.25783976912498474, + "fcm_dpo/margin": 19.849124908447266, + "fcm_dpo/q_t": 0.3747923970222473, + "grad_norm": 18.076324462890625, + "learning_rate": 3.3301533956555885e-07, + "logits/chosen": 0.6100283861160278, + "logits/rejected": 0.578555166721344, + "logps/chosen": -102.57945251464844, + "logps/ref_chosen": -52.85089111328125, + "logps/ref_rejected": -69.97584533691406, + "logps/rejected": -139.55352783203125, + "loss": 1.0996, + "margin_dpo/margin_mean": 19.84912872314453, + "margin_dpo/margin_std": 34.676185607910156, + "step": 301 + }, + { + "KL/chosen_KL_mean": -54.77104568481445, + "KL/mean": -61.995399475097656, + "KL/rejected_KL_mean": -69.21975708007812, + "KL/std": 30.460678100585938, + "epoch": 0.4565381708238851, + "fcm_dpo/beta": 0.03298091143369675, + "fcm_dpo/delta": -0.0769069567322731, + "fcm_dpo/margin": 14.448715209960938, + "fcm_dpo/q_t": 0.4056761860847473, + "grad_norm": 23.014057159423828, + "learning_rate": 3.317669908293554e-07, + "logits/chosen": 0.4416104257106781, + "logits/rejected": 0.38413500785827637, + "logps/chosen": -121.73755645751953, + "logps/ref_chosen": -66.96650695800781, + "logps/ref_rejected": -88.09510803222656, + "logps/rejected": -157.31488037109375, + "loss": 1.1955, + "margin_dpo/margin_mean": 14.448715209960938, + "margin_dpo/margin_std": 31.784908294677734, + "step": 302 + }, + { + "KL/chosen_KL_mean": -45.928810119628906, + "KL/mean": -59.719505310058594, + "KL/rejected_KL_mean": -73.51019287109375, + "KL/std": 31.779054641723633, + "epoch": 0.4580498866213152, + "fcm_dpo/beta": 0.032726868987083435, + "fcm_dpo/delta": -0.5071576237678528, + "fcm_dpo/margin": 27.581382751464844, + "fcm_dpo/q_t": 0.3330722153186798, + "grad_norm": 16.48186683654785, + "learning_rate": 3.3051635489464793e-07, + "logits/chosen": 0.5711244344711304, + "logits/rejected": 0.4999094605445862, + "logps/chosen": -108.05033874511719, + "logps/ref_chosen": -62.12152862548828, + "logps/ref_rejected": -90.31204223632812, + "logps/rejected": -163.82223510742188, + "loss": 0.963, + "margin_dpo/margin_mean": 27.581382751464844, + "margin_dpo/margin_std": 36.716556549072266, + "step": 303 + }, + { + "KL/chosen_KL_mean": -43.467010498046875, + "KL/mean": -57.39947509765625, + "KL/rejected_KL_mean": -71.33193969726562, + "KL/std": 34.39189910888672, + "epoch": 0.4595616024187453, + "fcm_dpo/beta": 0.03239506483078003, + "fcm_dpo/delta": -0.5072128772735596, + "fcm_dpo/margin": 27.864925384521484, + "fcm_dpo/q_t": 0.3185381293296814, + "grad_norm": 17.851577758789062, + "learning_rate": 3.292634667444117e-07, + "logits/chosen": 0.5406277775764465, + "logits/rejected": 0.4828648269176483, + "logps/chosen": -104.16210174560547, + "logps/ref_chosen": -60.695091247558594, + "logps/ref_rejected": -78.2525405883789, + "logps/rejected": -149.58447265625, + "loss": 0.8563, + "margin_dpo/margin_mean": 27.864927291870117, + "margin_dpo/margin_std": 29.170560836791992, + "step": 304 + }, + { + "KL/chosen_KL_mean": -50.7522087097168, + "KL/mean": -61.07701110839844, + "KL/rejected_KL_mean": -71.40182495117188, + "KL/std": 31.32061004638672, + "epoch": 0.46107331821617537, + "fcm_dpo/beta": 0.03215644508600235, + "fcm_dpo/delta": -0.2660941481590271, + "fcm_dpo/margin": 20.649612426757812, + "fcm_dpo/q_t": 0.37116116285324097, + "grad_norm": 22.04763412475586, + "learning_rate": 3.280083614246217e-07, + "logits/chosen": 0.5072367191314697, + "logits/rejected": 0.533474326133728, + "logps/chosen": -123.45134735107422, + "logps/ref_chosen": -72.69914245605469, + "logps/ref_rejected": -65.65670776367188, + "logps/rejected": -137.05853271484375, + "loss": 1.0966, + "margin_dpo/margin_mean": 20.649614334106445, + "margin_dpo/margin_std": 34.96246337890625, + "step": 305 + }, + { + "KL/chosen_KL_mean": -48.040496826171875, + "KL/mean": -58.037235260009766, + "KL/rejected_KL_mean": -68.03397369384766, + "KL/std": 29.662532806396484, + "epoch": 0.46258503401360546, + "fcm_dpo/beta": 0.03198698163032532, + "fcm_dpo/delta": -0.24156554043293, + "fcm_dpo/margin": 19.99347686767578, + "fcm_dpo/q_t": 0.36749911308288574, + "grad_norm": 17.132556915283203, + "learning_rate": 3.267510740432719e-07, + "logits/chosen": 0.6217135190963745, + "logits/rejected": 0.5042356252670288, + "logps/chosen": -102.01103210449219, + "logps/ref_chosen": -53.97052764892578, + "logps/ref_rejected": -71.02423095703125, + "logps/rejected": -139.05819702148438, + "loss": 1.0054, + "margin_dpo/margin_mean": 19.99347686767578, + "margin_dpo/margin_std": 26.61040496826172, + "step": 306 + }, + { + "KL/chosen_KL_mean": -41.22514343261719, + "KL/mean": -48.46742248535156, + "KL/rejected_KL_mean": -55.7097053527832, + "KL/std": 30.47110366821289, + "epoch": 0.46409674981103555, + "fcm_dpo/beta": 0.03196690231561661, + "fcm_dpo/delta": -0.06333252787590027, + "fcm_dpo/margin": 14.484560012817383, + "fcm_dpo/q_t": 0.41517025232315063, + "grad_norm": 20.775392532348633, + "learning_rate": 3.2549163976939285e-07, + "logits/chosen": 0.593315839767456, + "logits/rejected": 0.545011043548584, + "logps/chosen": -98.63825225830078, + "logps/ref_chosen": -57.413108825683594, + "logps/ref_rejected": -68.68010711669922, + "logps/rejected": -124.38981628417969, + "loss": 1.2469, + "margin_dpo/margin_mean": 14.484560012817383, + "margin_dpo/margin_std": 34.95436096191406, + "step": 307 + }, + { + "KL/chosen_KL_mean": -44.41400909423828, + "KL/mean": -54.375221252441406, + "KL/rejected_KL_mean": -64.33644104003906, + "KL/std": 31.222042083740234, + "epoch": 0.4656084656084656, + "fcm_dpo/beta": 0.03180667385458946, + "fcm_dpo/delta": -0.2354317307472229, + "fcm_dpo/margin": 19.922426223754883, + "fcm_dpo/q_t": 0.3721604347229004, + "grad_norm": 17.209348678588867, + "learning_rate": 3.2423009383206874e-07, + "logits/chosen": 0.5420068502426147, + "logits/rejected": 0.5284662842750549, + "logps/chosen": -111.01280212402344, + "logps/ref_chosen": -66.59879302978516, + "logps/ref_rejected": -74.337158203125, + "logps/rejected": -138.673583984375, + "loss": 1.0808, + "margin_dpo/margin_mean": 19.922426223754883, + "margin_dpo/margin_std": 32.33723831176758, + "step": 308 + }, + { + "KL/chosen_KL_mean": -49.61011505126953, + "KL/mean": -59.66952133178711, + "KL/rejected_KL_mean": -69.72892761230469, + "KL/std": 28.91370964050293, + "epoch": 0.4671201814058957, + "fcm_dpo/beta": 0.03165272995829582, + "fcm_dpo/delta": -0.2386629283428192, + "fcm_dpo/margin": 20.118816375732422, + "fcm_dpo/q_t": 0.372336208820343, + "grad_norm": 19.450109481811523, + "learning_rate": 3.229664715194511e-07, + "logits/chosen": 0.5981101989746094, + "logits/rejected": 0.5373047590255737, + "logps/chosen": -115.00485229492188, + "logps/ref_chosen": -65.39474487304688, + "logps/ref_rejected": -75.70930480957031, + "logps/rejected": -145.438232421875, + "loss": 1.0506, + "margin_dpo/margin_mean": 20.118816375732422, + "margin_dpo/margin_std": 30.500198364257812, + "step": 309 + }, + { + "KL/chosen_KL_mean": -49.98461151123047, + "KL/mean": -56.13789367675781, + "KL/rejected_KL_mean": -62.291175842285156, + "KL/std": 27.158016204833984, + "epoch": 0.46863189720332576, + "fcm_dpo/beta": 0.03163716197013855, + "fcm_dpo/delta": 0.010684855282306671, + "fcm_dpo/margin": 12.306563377380371, + "fcm_dpo/q_t": 0.4255636930465698, + "grad_norm": 20.57244873046875, + "learning_rate": 3.2170080817777257e-07, + "logits/chosen": 0.5837448835372925, + "logits/rejected": 0.568115234375, + "logps/chosen": -124.65288543701172, + "logps/ref_chosen": -74.66827392578125, + "logps/ref_rejected": -80.5689697265625, + "logps/rejected": -142.86013793945312, + "loss": 1.2455, + "margin_dpo/margin_mean": 12.306564331054688, + "margin_dpo/margin_std": 31.14623260498047, + "step": 310 + }, + { + "KL/chosen_KL_mean": -40.15812683105469, + "KL/mean": -50.16162872314453, + "KL/rejected_KL_mean": -60.165130615234375, + "KL/std": 32.771244049072266, + "epoch": 0.47014361300075586, + "fcm_dpo/beta": 0.031512945890426636, + "fcm_dpo/delta": -0.2322458028793335, + "fcm_dpo/margin": 20.007003784179688, + "fcm_dpo/q_t": 0.3764254152774811, + "grad_norm": 18.221084594726562, + "learning_rate": 3.204331392103574e-07, + "logits/chosen": 0.48371759057044983, + "logits/rejected": 0.3413756489753723, + "logps/chosen": -99.89616394042969, + "logps/ref_chosen": -59.738033294677734, + "logps/ref_rejected": -93.60757446289062, + "logps/rejected": -153.772705078125, + "loss": 1.0602, + "margin_dpo/margin_mean": 20.007003784179688, + "margin_dpo/margin_std": 32.39842224121094, + "step": 311 + }, + { + "KL/chosen_KL_mean": -41.49217987060547, + "KL/mean": -54.82445526123047, + "KL/rejected_KL_mean": -68.15673065185547, + "KL/std": 31.256023406982422, + "epoch": 0.47165532879818595, + "fcm_dpo/beta": 0.031304437667131424, + "fcm_dpo/delta": -0.4382626414299011, + "fcm_dpo/margin": 26.66455841064453, + "fcm_dpo/q_t": 0.3316800594329834, + "grad_norm": 15.331438064575195, + "learning_rate": 3.1916350007663176e-07, + "logits/chosen": 0.5764449834823608, + "logits/rejected": 0.4757081866264343, + "logps/chosen": -95.30860900878906, + "logps/ref_chosen": -53.816436767578125, + "logps/ref_rejected": -68.6575698852539, + "logps/rejected": -136.81430053710938, + "loss": 0.8958, + "margin_dpo/margin_mean": 26.66455841064453, + "margin_dpo/margin_std": 29.972898483276367, + "step": 312 + }, + { + "KL/chosen_KL_mean": -41.31776809692383, + "KL/mean": -49.32701873779297, + "KL/rejected_KL_mean": -57.336265563964844, + "KL/std": 30.415565490722656, + "epoch": 0.47316704459561604, + "fcm_dpo/beta": 0.031179986894130707, + "fcm_dpo/delta": -0.09995320439338684, + "fcm_dpo/margin": 16.018491744995117, + "fcm_dpo/q_t": 0.4042476415634155, + "grad_norm": 18.701175689697266, + "learning_rate": 3.178919262911314e-07, + "logits/chosen": 0.6190089583396912, + "logits/rejected": 0.5964229106903076, + "logps/chosen": -101.27513122558594, + "logps/ref_chosen": -59.957359313964844, + "logps/ref_rejected": -69.31729888916016, + "logps/rejected": -126.653564453125, + "loss": 1.1702, + "margin_dpo/margin_mean": 16.018489837646484, + "margin_dpo/margin_std": 33.22297668457031, + "step": 313 + }, + { + "KL/chosen_KL_mean": -38.77689743041992, + "KL/mean": -51.97898864746094, + "KL/rejected_KL_mean": -65.18107604980469, + "KL/std": 29.90279769897461, + "epoch": 0.47467876039304613, + "fcm_dpo/beta": 0.03096053935587406, + "fcm_dpo/delta": -0.42103683948516846, + "fcm_dpo/margin": 26.4041748046875, + "fcm_dpo/q_t": 0.33664670586586, + "grad_norm": 15.793561935424805, + "learning_rate": 3.166184534225087e-07, + "logits/chosen": 0.5692810416221619, + "logits/rejected": 0.59623783826828, + "logps/chosen": -109.04505920410156, + "logps/ref_chosen": -70.26815795898438, + "logps/ref_rejected": -69.23971557617188, + "logps/rejected": -134.42079162597656, + "loss": 0.8997, + "margin_dpo/margin_mean": 26.404176712036133, + "margin_dpo/margin_std": 29.464202880859375, + "step": 314 + }, + { + "KL/chosen_KL_mean": -40.42737579345703, + "KL/mean": -51.105224609375, + "KL/rejected_KL_mean": -61.783077239990234, + "KL/std": 29.70652198791504, + "epoch": 0.47619047619047616, + "fcm_dpo/beta": 0.030817130580544472, + "fcm_dpo/delta": -0.2597944438457489, + "fcm_dpo/margin": 21.3557071685791, + "fcm_dpo/q_t": 0.3625257611274719, + "grad_norm": 15.870321273803711, + "learning_rate": 3.1534311709253723e-07, + "logits/chosen": 0.4995626211166382, + "logits/rejected": 0.45825695991516113, + "logps/chosen": -108.2220687866211, + "logps/ref_chosen": -67.79469299316406, + "logps/ref_rejected": -74.55148315429688, + "logps/rejected": -136.33456420898438, + "loss": 1.0044, + "margin_dpo/margin_mean": 21.3557071685791, + "margin_dpo/margin_std": 27.90345001220703, + "step": 315 + }, + { + "KL/chosen_KL_mean": -38.910579681396484, + "KL/mean": -52.10319519042969, + "KL/rejected_KL_mean": -65.29581451416016, + "KL/std": 30.841766357421875, + "epoch": 0.47770219198790626, + "fcm_dpo/beta": 0.030605999752879143, + "fcm_dpo/delta": -0.41074857115745544, + "fcm_dpo/margin": 26.385231018066406, + "fcm_dpo/q_t": 0.33775946497917175, + "grad_norm": 16.307104110717773, + "learning_rate": 3.1406595297511564e-07, + "logits/chosen": 0.45677274465560913, + "logits/rejected": 0.32220372557640076, + "logps/chosen": -94.19906616210938, + "logps/ref_chosen": -55.288482666015625, + "logps/ref_rejected": -96.15723419189453, + "logps/rejected": -161.4530487060547, + "loss": 0.9141, + "margin_dpo/margin_mean": 26.385231018066406, + "margin_dpo/margin_std": 28.32827377319336, + "step": 316 + }, + { + "KL/chosen_KL_mean": -34.75751495361328, + "KL/mean": -49.58986282348633, + "KL/rejected_KL_mean": -64.42221069335938, + "KL/std": 30.769298553466797, + "epoch": 0.47921390778533635, + "fcm_dpo/beta": 0.030244119465351105, + "fcm_dpo/delta": -0.5020566582679749, + "fcm_dpo/margin": 29.664697647094727, + "fcm_dpo/q_t": 0.32022345066070557, + "grad_norm": 17.66897964477539, + "learning_rate": 3.1278699679526975e-07, + "logits/chosen": 0.6189926862716675, + "logits/rejected": 0.5698972940444946, + "logps/chosen": -89.3388900756836, + "logps/ref_chosen": -54.58137512207031, + "logps/ref_rejected": -72.77232360839844, + "logps/rejected": -137.1945343017578, + "loss": 0.8534, + "margin_dpo/margin_mean": 29.66469955444336, + "margin_dpo/margin_std": 30.635875701904297, + "step": 317 + }, + { + "KL/chosen_KL_mean": -40.76923370361328, + "KL/mean": -50.604400634765625, + "KL/rejected_KL_mean": -60.43956756591797, + "KL/std": 31.436973571777344, + "epoch": 0.48072562358276644, + "fcm_dpo/beta": 0.030093831941485405, + "fcm_dpo/delta": -0.19313585758209229, + "fcm_dpo/margin": 19.670337677001953, + "fcm_dpo/q_t": 0.3901757001876831, + "grad_norm": 18.606689453125, + "learning_rate": 3.1150628432815336e-07, + "logits/chosen": 0.6018540263175964, + "logits/rejected": 0.5300246477127075, + "logps/chosen": -93.65746307373047, + "logps/ref_chosen": -52.88822937011719, + "logps/ref_rejected": -80.63988494873047, + "logps/rejected": -141.07945251464844, + "loss": 1.1862, + "margin_dpo/margin_mean": 19.670339584350586, + "margin_dpo/margin_std": 40.38948059082031, + "step": 318 + }, + { + "KL/chosen_KL_mean": -37.450836181640625, + "KL/mean": -51.321319580078125, + "KL/rejected_KL_mean": -65.19180297851562, + "KL/std": 32.808685302734375, + "epoch": 0.48223733938019653, + "fcm_dpo/beta": 0.029874565079808235, + "fcm_dpo/delta": -0.43234699964523315, + "fcm_dpo/margin": 27.740966796875, + "fcm_dpo/q_t": 0.34197819232940674, + "grad_norm": 17.177762985229492, + "learning_rate": 3.1022385139804707e-07, + "logits/chosen": 0.5362778306007385, + "logits/rejected": 0.5168346762657166, + "logps/chosen": -101.81416320800781, + "logps/ref_chosen": -64.36333465576172, + "logps/ref_rejected": -79.47296142578125, + "logps/rejected": -144.66476440429688, + "loss": 0.9415, + "margin_dpo/margin_mean": 27.740968704223633, + "margin_dpo/margin_std": 35.664146423339844, + "step": 319 + }, + { + "KL/chosen_KL_mean": -37.64230728149414, + "KL/mean": -47.92146682739258, + "KL/rejected_KL_mean": -58.200626373291016, + "KL/std": 33.739864349365234, + "epoch": 0.4837490551776266, + "fcm_dpo/beta": 0.029760433360934258, + "fcm_dpo/delta": -0.21326178312301636, + "fcm_dpo/margin": 20.558324813842773, + "fcm_dpo/q_t": 0.38366615772247314, + "grad_norm": 18.528676986694336, + "learning_rate": 3.0893973387735683e-07, + "logits/chosen": 0.4550350308418274, + "logits/rejected": 0.41082847118377686, + "logps/chosen": -87.2010498046875, + "logps/ref_chosen": -49.558746337890625, + "logps/ref_rejected": -71.23444366455078, + "logps/rejected": -129.43507385253906, + "loss": 1.099, + "margin_dpo/margin_mean": 20.55832290649414, + "margin_dpo/margin_std": 34.81462860107422, + "step": 320 + }, + { + "KL/chosen_KL_mean": -39.206634521484375, + "KL/mean": -50.471561431884766, + "KL/rejected_KL_mean": -61.736488342285156, + "KL/std": 31.298999786376953, + "epoch": 0.4852607709750567, + "fcm_dpo/beta": 0.029537076130509377, + "fcm_dpo/delta": -0.26767703890800476, + "fcm_dpo/margin": 22.52985382080078, + "fcm_dpo/q_t": 0.3650682866573334, + "grad_norm": 19.921937942504883, + "learning_rate": 3.0765396768561004e-07, + "logits/chosen": 0.5506191849708557, + "logits/rejected": 0.5294814109802246, + "logps/chosen": -91.29190063476562, + "logps/ref_chosen": -52.08526611328125, + "logps/ref_rejected": -55.58674621582031, + "logps/rejected": -117.32323455810547, + "loss": 1.0576, + "margin_dpo/margin_mean": 22.52985382080078, + "margin_dpo/margin_std": 34.42205810546875, + "step": 321 + }, + { + "KL/chosen_KL_mean": -43.35811233520508, + "KL/mean": -58.52265930175781, + "KL/rejected_KL_mean": -73.68720245361328, + "KL/std": 32.806243896484375, + "epoch": 0.48677248677248675, + "fcm_dpo/beta": 0.02932625636458397, + "fcm_dpo/delta": -0.49360156059265137, + "fcm_dpo/margin": 30.32908821105957, + "fcm_dpo/q_t": 0.32084935903549194, + "grad_norm": 17.190349578857422, + "learning_rate": 3.063665887884511e-07, + "logits/chosen": 0.5712053775787354, + "logits/rejected": 0.48572826385498047, + "logps/chosen": -90.76222229003906, + "logps/ref_chosen": -47.404109954833984, + "logps/ref_rejected": -73.4260025024414, + "logps/rejected": -147.1132049560547, + "loss": 0.8607, + "margin_dpo/margin_mean": 30.329086303710938, + "margin_dpo/margin_std": 31.72441864013672, + "step": 322 + }, + { + "KL/chosen_KL_mean": -44.81703186035156, + "KL/mean": -55.617393493652344, + "KL/rejected_KL_mean": -66.41775512695312, + "KL/std": 33.91140365600586, + "epoch": 0.48828420256991684, + "fcm_dpo/beta": 0.029152877628803253, + "fcm_dpo/delta": -0.2311132550239563, + "fcm_dpo/margin": 21.600723266601562, + "fcm_dpo/q_t": 0.3867204189300537, + "grad_norm": 18.1402587890625, + "learning_rate": 3.0507763319663517e-07, + "logits/chosen": 0.4958229064941406, + "logits/rejected": 0.4143955707550049, + "logps/chosen": -114.82333374023438, + "logps/ref_chosen": -70.00630187988281, + "logps/ref_rejected": -86.96690368652344, + "logps/rejected": -153.38465881347656, + "loss": 1.1308, + "margin_dpo/margin_mean": 21.600723266601562, + "margin_dpo/margin_std": 40.15673065185547, + "step": 323 + }, + { + "KL/chosen_KL_mean": -39.21222686767578, + "KL/mean": -52.04484558105469, + "KL/rejected_KL_mean": -64.8774642944336, + "KL/std": 34.94654083251953, + "epoch": 0.4897959183673469, + "fcm_dpo/beta": 0.02893686294555664, + "fcm_dpo/delta": -0.3454386591911316, + "fcm_dpo/margin": 25.665225982666016, + "fcm_dpo/q_t": 0.34842967987060547, + "grad_norm": 20.208866119384766, + "learning_rate": 3.0378713696502097e-07, + "logits/chosen": 0.5958482027053833, + "logits/rejected": 0.5353559851646423, + "logps/chosen": -95.10104370117188, + "logps/ref_chosen": -55.88882064819336, + "logps/ref_rejected": -75.23088073730469, + "logps/rejected": -140.10833740234375, + "loss": 0.9435, + "margin_dpo/margin_mean": 25.665225982666016, + "margin_dpo/margin_std": 31.14936065673828, + "step": 324 + }, + { + "KL/chosen_KL_mean": -48.04041290283203, + "KL/mean": -60.82289123535156, + "KL/rejected_KL_mean": -73.60537719726562, + "KL/std": 34.35237121582031, + "epoch": 0.491307634164777, + "fcm_dpo/beta": 0.028726819902658463, + "fcm_dpo/delta": -0.33738335967063904, + "fcm_dpo/margin": 25.56496238708496, + "fcm_dpo/q_t": 0.3560935854911804, + "grad_norm": 17.880403518676758, + "learning_rate": 3.0249513619156206e-07, + "logits/chosen": 0.5490873456001282, + "logits/rejected": 0.4810106158256531, + "logps/chosen": -112.18743133544922, + "logps/ref_chosen": -64.14701843261719, + "logps/ref_rejected": -79.91143798828125, + "logps/rejected": -153.51681518554688, + "loss": 1.0255, + "margin_dpo/margin_mean": 25.564964294433594, + "margin_dpo/margin_std": 38.03911590576172, + "step": 325 + }, + { + "KL/chosen_KL_mean": -56.850563049316406, + "KL/mean": -63.5693244934082, + "KL/rejected_KL_mean": -70.2880859375, + "KL/std": 33.31139373779297, + "epoch": 0.4928193499622071, + "fcm_dpo/beta": 0.028705760836601257, + "fcm_dpo/delta": 0.014314810745418072, + "fcm_dpo/margin": 13.437520027160645, + "fcm_dpo/q_t": 0.42071533203125, + "grad_norm": 22.36930274963379, + "learning_rate": 3.012016670162977e-07, + "logits/chosen": 0.4895935654640198, + "logits/rejected": 0.49452024698257446, + "logps/chosen": -132.38186645507812, + "logps/ref_chosen": -75.53131103515625, + "logps/ref_rejected": -76.5898666381836, + "logps/rejected": -146.87796020507812, + "loss": 1.2671, + "margin_dpo/margin_mean": 13.437520027160645, + "margin_dpo/margin_std": 35.288917541503906, + "step": 326 + }, + { + "KL/chosen_KL_mean": -52.766849517822266, + "KL/mean": -63.28242111206055, + "KL/rejected_KL_mean": -73.79798889160156, + "KL/std": 35.24690246582031, + "epoch": 0.4943310657596372, + "fcm_dpo/beta": 0.02863834798336029, + "fcm_dpo/delta": -0.20347318053245544, + "fcm_dpo/margin": 21.031137466430664, + "fcm_dpo/q_t": 0.3840605318546295, + "grad_norm": 21.609195709228516, + "learning_rate": 2.99906765620341e-07, + "logits/chosen": 0.4372777044773102, + "logits/rejected": 0.4017803966999054, + "logps/chosen": -122.10401916503906, + "logps/ref_chosen": -69.33717346191406, + "logps/ref_rejected": -73.37751770019531, + "logps/rejected": -147.17550659179688, + "loss": 1.1387, + "margin_dpo/margin_mean": 21.031137466430664, + "margin_dpo/margin_std": 39.46784210205078, + "step": 327 + }, + { + "KL/chosen_KL_mean": -43.75243377685547, + "KL/mean": -56.70161056518555, + "KL/rejected_KL_mean": -69.65078735351562, + "KL/std": 35.66333770751953, + "epoch": 0.4958427815570673, + "fcm_dpo/beta": 0.028457406908273697, + "fcm_dpo/delta": -0.3394482731819153, + "fcm_dpo/margin": 25.898353576660156, + "fcm_dpo/q_t": 0.3530447781085968, + "grad_norm": 17.121482849121094, + "learning_rate": 2.9861046822486766e-07, + "logits/chosen": 0.5128360986709595, + "logits/rejected": 0.4763758182525635, + "logps/chosen": -105.45866394042969, + "logps/ref_chosen": -61.70623016357422, + "logps/ref_rejected": -83.73808288574219, + "logps/rejected": -153.3888702392578, + "loss": 0.9505, + "margin_dpo/margin_mean": 25.898351669311523, + "margin_dpo/margin_std": 32.10985565185547, + "step": 328 + }, + { + "KL/chosen_KL_mean": -51.539737701416016, + "KL/mean": -65.16473388671875, + "KL/rejected_KL_mean": -78.78971862792969, + "KL/std": 36.741641998291016, + "epoch": 0.4973544973544973, + "fcm_dpo/beta": 0.028234798461198807, + "fcm_dpo/delta": -0.3724392056465149, + "fcm_dpo/margin": 27.249984741210938, + "fcm_dpo/q_t": 0.3530978262424469, + "grad_norm": 19.202136993408203, + "learning_rate": 2.9731281109010253e-07, + "logits/chosen": 0.6009070873260498, + "logits/rejected": 0.5414671301841736, + "logps/chosen": -116.03814697265625, + "logps/ref_chosen": -64.4984130859375, + "logps/ref_rejected": -83.6591796875, + "logps/rejected": -162.4488983154297, + "loss": 0.9735, + "margin_dpo/margin_mean": 27.24998664855957, + "margin_dpo/margin_std": 36.722267150878906, + "step": 329 + }, + { + "KL/chosen_KL_mean": -44.201045989990234, + "KL/mean": -57.723876953125, + "KL/rejected_KL_mean": -71.24669647216797, + "KL/std": 35.23374557495117, + "epoch": 0.4988662131519274, + "fcm_dpo/beta": 0.028062237426638603, + "fcm_dpo/delta": -0.36154794692993164, + "fcm_dpo/margin": 27.045654296875, + "fcm_dpo/q_t": 0.35366714000701904, + "grad_norm": 17.607675552368164, + "learning_rate": 2.9601383051430505e-07, + "logits/chosen": 0.5492699146270752, + "logits/rejected": 0.4775303602218628, + "logps/chosen": -99.00569152832031, + "logps/ref_chosen": -54.80464172363281, + "logps/ref_rejected": -75.3194351196289, + "logps/rejected": -146.56613159179688, + "loss": 1.0568, + "margin_dpo/margin_mean": 27.045654296875, + "margin_dpo/margin_std": 42.25658416748047, + "step": 330 + }, + { + "KL/chosen_KL_mean": -47.843536376953125, + "KL/mean": -66.39089965820312, + "KL/rejected_KL_mean": -84.93826293945312, + "KL/std": 39.887325286865234, + "epoch": 0.5003779289493575, + "fcm_dpo/beta": 0.027729129418730736, + "fcm_dpo/delta": -0.6350520849227905, + "fcm_dpo/margin": 37.094730377197266, + "fcm_dpo/q_t": 0.3087009787559509, + "grad_norm": 16.450267791748047, + "learning_rate": 2.947135628327544e-07, + "logits/chosen": 0.6751506328582764, + "logits/rejected": 0.6446682214736938, + "logps/chosen": -107.08612060546875, + "logps/ref_chosen": -59.242584228515625, + "logps/ref_rejected": -69.87483215332031, + "logps/rejected": -154.81309509277344, + "loss": 0.8868, + "margin_dpo/margin_mean": 37.094730377197266, + "margin_dpo/margin_std": 43.963722229003906, + "step": 331 + }, + { + "KL/chosen_KL_mean": -50.796607971191406, + "KL/mean": -65.41808319091797, + "KL/rejected_KL_mean": -80.03956604003906, + "KL/std": 36.960784912109375, + "epoch": 0.5018896447467877, + "fcm_dpo/beta": 0.027502547949552536, + "fcm_dpo/delta": -0.40733611583709717, + "fcm_dpo/margin": 29.24295425415039, + "fcm_dpo/q_t": 0.34708938002586365, + "grad_norm": 17.83302116394043, + "learning_rate": 2.934120444167326e-07, + "logits/chosen": 0.522958517074585, + "logits/rejected": 0.47490304708480835, + "logps/chosen": -117.90636444091797, + "logps/ref_chosen": -67.10975646972656, + "logps/ref_rejected": -77.11839294433594, + "logps/rejected": -157.157958984375, + "loss": 0.9416, + "margin_dpo/margin_mean": 29.24295425415039, + "margin_dpo/margin_std": 36.47911834716797, + "step": 332 + }, + { + "KL/chosen_KL_mean": -53.729087829589844, + "KL/mean": -70.67256927490234, + "KL/rejected_KL_mean": -87.61605834960938, + "KL/std": 37.699485778808594, + "epoch": 0.5034013605442177, + "fcm_dpo/beta": 0.027205023914575577, + "fcm_dpo/delta": -0.5266174674034119, + "fcm_dpo/margin": 33.88695526123047, + "fcm_dpo/q_t": 0.3205938935279846, + "grad_norm": 14.599785804748535, + "learning_rate": 2.921093116725076e-07, + "logits/chosen": 0.575968861579895, + "logits/rejected": 0.4990413784980774, + "logps/chosen": -112.11021423339844, + "logps/ref_chosen": -58.381134033203125, + "logps/ref_rejected": -85.02839660644531, + "logps/rejected": -172.64443969726562, + "loss": 0.8549, + "margin_dpo/margin_mean": 33.886959075927734, + "margin_dpo/margin_std": 36.588462829589844, + "step": 333 + }, + { + "KL/chosen_KL_mean": -53.68241500854492, + "KL/mean": -65.27969360351562, + "KL/rejected_KL_mean": -76.8769760131836, + "KL/std": 40.12836456298828, + "epoch": 0.5049130763416477, + "fcm_dpo/beta": 0.027033882215619087, + "fcm_dpo/delta": -0.22846141457557678, + "fcm_dpo/margin": 23.194564819335938, + "fcm_dpo/q_t": 0.381631076335907, + "grad_norm": 20.677330017089844, + "learning_rate": 2.9080540104031484e-07, + "logits/chosen": 0.6133550405502319, + "logits/rejected": 0.5640695095062256, + "logps/chosen": -120.57441711425781, + "logps/ref_chosen": -66.89199829101562, + "logps/ref_rejected": -91.83695220947266, + "logps/rejected": -168.71392822265625, + "loss": 1.1378, + "margin_dpo/margin_mean": 23.194568634033203, + "margin_dpo/margin_std": 43.62317657470703, + "step": 334 + }, + { + "KL/chosen_KL_mean": -54.18124008178711, + "KL/mean": -67.98838806152344, + "KL/rejected_KL_mean": -81.79554748535156, + "KL/std": 39.49427032470703, + "epoch": 0.5064247921390779, + "fcm_dpo/beta": 0.026879791170358658, + "fcm_dpo/delta": -0.3446880578994751, + "fcm_dpo/margin": 27.614307403564453, + "fcm_dpo/q_t": 0.36386793851852417, + "grad_norm": 23.50693702697754, + "learning_rate": 2.895003489933375e-07, + "logits/chosen": 0.5958684682846069, + "logits/rejected": 0.5559037327766418, + "logps/chosen": -115.69569396972656, + "logps/ref_chosen": -61.51445770263672, + "logps/ref_rejected": -75.68916320800781, + "logps/rejected": -157.48471069335938, + "loss": 1.0507, + "margin_dpo/margin_mean": 27.614303588867188, + "margin_dpo/margin_std": 43.3466796875, + "step": 335 + }, + { + "KL/chosen_KL_mean": -65.34312438964844, + "KL/mean": -78.89384460449219, + "KL/rejected_KL_mean": -92.44454956054688, + "KL/std": 40.68659591674805, + "epoch": 0.5079365079365079, + "fcm_dpo/beta": 0.026655372232198715, + "fcm_dpo/delta": -0.3253745138645172, + "fcm_dpo/margin": 27.101417541503906, + "fcm_dpo/q_t": 0.37097546458244324, + "grad_norm": 19.950565338134766, + "learning_rate": 2.8819419203668675e-07, + "logits/chosen": 0.5154187679290771, + "logits/rejected": 0.49114304780960083, + "logps/chosen": -134.1931915283203, + "logps/ref_chosen": -68.85006713867188, + "logps/ref_rejected": -92.99603271484375, + "logps/rejected": -185.44058227539062, + "loss": 1.0541, + "margin_dpo/margin_mean": 27.101417541503906, + "margin_dpo/margin_std": 43.549278259277344, + "step": 336 + }, + { + "KL/chosen_KL_mean": -65.30525970458984, + "KL/mean": -76.23297119140625, + "KL/rejected_KL_mean": -87.16069030761719, + "KL/std": 39.9510498046875, + "epoch": 0.509448223733938, + "fcm_dpo/beta": 0.026574313640594482, + "fcm_dpo/delta": -0.18182379007339478, + "fcm_dpo/margin": 21.855432510375977, + "fcm_dpo/q_t": 0.38537168502807617, + "grad_norm": 19.033676147460938, + "learning_rate": 2.8688696670638053e-07, + "logits/chosen": 0.456851065158844, + "logits/rejected": 0.42343103885650635, + "logps/chosen": -138.49310302734375, + "logps/ref_chosen": -73.18783569335938, + "logps/ref_rejected": -86.89118957519531, + "logps/rejected": -174.0518798828125, + "loss": 1.12, + "margin_dpo/margin_mean": 21.85543441772461, + "margin_dpo/margin_std": 39.6829833984375, + "step": 337 + }, + { + "KL/chosen_KL_mean": -65.23648834228516, + "KL/mean": -76.54876708984375, + "KL/rejected_KL_mean": -87.86103820800781, + "KL/std": 38.92296600341797, + "epoch": 0.5109599395313681, + "fcm_dpo/beta": 0.026467353105545044, + "fcm_dpo/delta": -0.20000049471855164, + "fcm_dpo/margin": 22.624557495117188, + "fcm_dpo/q_t": 0.38381096720695496, + "grad_norm": 22.04119110107422, + "learning_rate": 2.8557870956832133e-07, + "logits/chosen": 0.5582261681556702, + "logits/rejected": 0.529557466506958, + "logps/chosen": -129.1761016845703, + "logps/ref_chosen": -63.939613342285156, + "logps/ref_rejected": -75.34243774414062, + "logps/rejected": -163.2034912109375, + "loss": 1.1197, + "margin_dpo/margin_mean": 22.624557495117188, + "margin_dpo/margin_std": 40.99917984008789, + "step": 338 + }, + { + "KL/chosen_KL_mean": -54.107017517089844, + "KL/mean": -67.47621154785156, + "KL/rejected_KL_mean": -80.84542083740234, + "KL/std": 37.78194046020508, + "epoch": 0.5124716553287982, + "fcm_dpo/beta": 0.02630516141653061, + "fcm_dpo/delta": -0.3057170510292053, + "fcm_dpo/margin": 26.738399505615234, + "fcm_dpo/q_t": 0.36096668243408203, + "grad_norm": 19.143558502197266, + "learning_rate": 2.842694572172736e-07, + "logits/chosen": 0.703507661819458, + "logits/rejected": 0.6146845817565918, + "logps/chosen": -99.65614318847656, + "logps/ref_chosen": -45.54913330078125, + "logps/ref_rejected": -67.0482177734375, + "logps/rejected": -147.89364624023438, + "loss": 1.0123, + "margin_dpo/margin_mean": 26.738399505615234, + "margin_dpo/margin_std": 37.722721099853516, + "step": 339 + }, + { + "KL/chosen_KL_mean": -61.39227294921875, + "KL/mean": -75.74258422851562, + "KL/rejected_KL_mean": -90.09290313720703, + "KL/std": 42.998802185058594, + "epoch": 0.5139833711262283, + "fcm_dpo/beta": 0.026158850640058517, + "fcm_dpo/delta": -0.35327795147895813, + "fcm_dpo/margin": 28.70063018798828, + "fcm_dpo/q_t": 0.36624759435653687, + "grad_norm": 19.922765731811523, + "learning_rate": 2.8295924627584004e-07, + "logits/chosen": 0.5988013744354248, + "logits/rejected": 0.5743155479431152, + "logps/chosen": -115.39791870117188, + "logps/ref_chosen": -54.00564956665039, + "logps/ref_rejected": -61.314430236816406, + "logps/rejected": -151.40733337402344, + "loss": 1.0824, + "margin_dpo/margin_mean": 28.70063018798828, + "margin_dpo/margin_std": 47.50862121582031, + "step": 340 + }, + { + "KL/chosen_KL_mean": -60.093475341796875, + "KL/mean": -75.38809204101562, + "KL/rejected_KL_mean": -90.68269348144531, + "KL/std": 41.160377502441406, + "epoch": 0.5154950869236583, + "fcm_dpo/beta": 0.02589040994644165, + "fcm_dpo/delta": -0.3965362310409546, + "fcm_dpo/margin": 30.589221954345703, + "fcm_dpo/q_t": 0.3502144515514374, + "grad_norm": 20.531843185424805, + "learning_rate": 2.816481133934373e-07, + "logits/chosen": 0.5874067544937134, + "logits/rejected": 0.5361970067024231, + "logps/chosen": -123.48857116699219, + "logps/ref_chosen": -63.39509582519531, + "logps/ref_rejected": -76.20973205566406, + "logps/rejected": -166.89242553710938, + "loss": 1.015, + "margin_dpo/margin_mean": 30.589221954345703, + "margin_dpo/margin_std": 42.35905838012695, + "step": 341 + }, + { + "KL/chosen_KL_mean": -58.60978698730469, + "KL/mean": -75.58102416992188, + "KL/rejected_KL_mean": -92.5522689819336, + "KL/std": 43.24871826171875, + "epoch": 0.5170068027210885, + "fcm_dpo/beta": 0.025693554431200027, + "fcm_dpo/delta": -0.476462721824646, + "fcm_dpo/margin": 33.942481994628906, + "fcm_dpo/q_t": 0.3395392894744873, + "grad_norm": 16.138137817382812, + "learning_rate": 2.8033609524527046e-07, + "logits/chosen": 0.6125584840774536, + "logits/rejected": 0.5693821310997009, + "logps/chosen": -111.65760803222656, + "logps/ref_chosen": -53.047813415527344, + "logps/ref_rejected": -68.2854232788086, + "logps/rejected": -160.8376922607422, + "loss": 0.9322, + "margin_dpo/margin_mean": 33.942481994628906, + "margin_dpo/margin_std": 43.357303619384766, + "step": 342 + }, + { + "KL/chosen_KL_mean": -56.86198425292969, + "KL/mean": -66.71636962890625, + "KL/rejected_KL_mean": -76.57075500488281, + "KL/std": 39.91425323486328, + "epoch": 0.5185185185185185, + "fcm_dpo/beta": 0.025541264563798904, + "fcm_dpo/delta": -0.19619542360305786, + "fcm_dpo/margin": 19.708772659301758, + "fcm_dpo/q_t": 0.3964730501174927, + "grad_norm": 18.149141311645508, + "learning_rate": 2.7902322853130753e-07, + "logits/chosen": 0.4626293182373047, + "logits/rejected": 0.45302748680114746, + "logps/chosen": -127.44051361083984, + "logps/ref_chosen": -70.57852935791016, + "logps/ref_rejected": -84.73873901367188, + "logps/rejected": -161.3094940185547, + "loss": 1.1491, + "margin_dpo/margin_mean": 19.708772659301758, + "margin_dpo/margin_std": 36.98005294799805, + "step": 343 + }, + { + "KL/chosen_KL_mean": -63.30127716064453, + "KL/mean": -78.61820220947266, + "KL/rejected_KL_mean": -93.93512725830078, + "KL/std": 40.37184143066406, + "epoch": 0.5200302343159486, + "fcm_dpo/beta": 0.025381244719028473, + "fcm_dpo/delta": -0.38077330589294434, + "fcm_dpo/margin": 30.63386344909668, + "fcm_dpo/q_t": 0.3483344316482544, + "grad_norm": 18.35032081604004, + "learning_rate": 2.7770954997525274e-07, + "logits/chosen": 0.5928553342819214, + "logits/rejected": 0.5235949754714966, + "logps/chosen": -119.1122817993164, + "logps/ref_chosen": -55.811004638671875, + "logps/ref_rejected": -84.77637481689453, + "logps/rejected": -178.7115020751953, + "loss": 0.9734, + "margin_dpo/margin_mean": 30.633861541748047, + "margin_dpo/margin_std": 40.95970153808594, + "step": 344 + }, + { + "KL/chosen_KL_mean": -48.310203552246094, + "KL/mean": -62.957275390625, + "KL/rejected_KL_mean": -77.6043472290039, + "KL/std": 37.735633850097656, + "epoch": 0.5215419501133787, + "fcm_dpo/beta": 0.025218918919563293, + "fcm_dpo/delta": -0.34127917885780334, + "fcm_dpo/margin": 29.294139862060547, + "fcm_dpo/q_t": 0.35675299167633057, + "grad_norm": 18.649444580078125, + "learning_rate": 2.7639509632351927e-07, + "logits/chosen": 0.6467137336730957, + "logits/rejected": 0.5995541214942932, + "logps/chosen": -106.09629821777344, + "logps/ref_chosen": -57.78609848022461, + "logps/ref_rejected": -78.91847229003906, + "logps/rejected": -156.5228271484375, + "loss": 1.0244, + "margin_dpo/margin_mean": 29.294139862060547, + "margin_dpo/margin_std": 43.7929573059082, + "step": 345 + }, + { + "KL/chosen_KL_mean": -52.95964431762695, + "KL/mean": -70.20674133300781, + "KL/rejected_KL_mean": -87.45382690429688, + "KL/std": 42.024349212646484, + "epoch": 0.5230536659108088, + "fcm_dpo/beta": 0.025016264989972115, + "fcm_dpo/delta": -0.4667249321937561, + "fcm_dpo/margin": 34.49418640136719, + "fcm_dpo/q_t": 0.3371295928955078, + "grad_norm": 17.98917007446289, + "learning_rate": 2.7507990434420123e-07, + "logits/chosen": 0.6152539849281311, + "logits/rejected": 0.5311729311943054, + "logps/chosen": -109.24476623535156, + "logps/ref_chosen": -56.285125732421875, + "logps/ref_rejected": -91.15303039550781, + "logps/rejected": -178.60687255859375, + "loss": 0.9664, + "margin_dpo/margin_mean": 34.49418640136719, + "margin_dpo/margin_std": 45.426918029785156, + "step": 346 + }, + { + "KL/chosen_KL_mean": -57.9916877746582, + "KL/mean": -70.969970703125, + "KL/rejected_KL_mean": -83.94825744628906, + "KL/std": 42.39558410644531, + "epoch": 0.5245653817082389, + "fcm_dpo/beta": 0.024852270260453224, + "fcm_dpo/delta": -0.24664340913295746, + "fcm_dpo/margin": 25.95656967163086, + "fcm_dpo/q_t": 0.37607258558273315, + "grad_norm": 21.636096954345703, + "learning_rate": 2.737640108260456e-07, + "logits/chosen": 0.7026286721229553, + "logits/rejected": 0.6515735387802124, + "logps/chosen": -111.49122619628906, + "logps/ref_chosen": -53.499542236328125, + "logps/ref_rejected": -72.52565002441406, + "logps/rejected": -156.47390747070312, + "loss": 1.0585, + "margin_dpo/margin_mean": 25.95656967163086, + "margin_dpo/margin_std": 41.366539001464844, + "step": 347 + }, + { + "KL/chosen_KL_mean": -51.39307403564453, + "KL/mean": -67.70439910888672, + "KL/rejected_KL_mean": -84.01571655273438, + "KL/std": 40.55320739746094, + "epoch": 0.5260770975056689, + "fcm_dpo/beta": 0.024652201682329178, + "fcm_dpo/delta": -0.4078223705291748, + "fcm_dpo/margin": 32.622642517089844, + "fcm_dpo/q_t": 0.3582861125469208, + "grad_norm": 18.745269775390625, + "learning_rate": 2.724474525774229e-07, + "logits/chosen": 0.7095910310745239, + "logits/rejected": 0.6775566339492798, + "logps/chosen": -102.179931640625, + "logps/ref_chosen": -50.78684997558594, + "logps/ref_rejected": -68.63732147216797, + "logps/rejected": -152.65304565429688, + "loss": 1.0143, + "margin_dpo/margin_mean": 32.62264633178711, + "margin_dpo/margin_std": 48.74144744873047, + "step": 348 + }, + { + "KL/chosen_KL_mean": -51.25471115112305, + "KL/mean": -66.97186279296875, + "KL/rejected_KL_mean": -82.68901824951172, + "KL/std": 42.62870407104492, + "epoch": 0.527588813303099, + "fcm_dpo/beta": 0.024495067074894905, + "fcm_dpo/delta": -0.372702419757843, + "fcm_dpo/margin": 31.43431282043457, + "fcm_dpo/q_t": 0.3527294993400574, + "grad_norm": 17.3203125, + "learning_rate": 2.711302664252973e-07, + "logits/chosen": 0.619326114654541, + "logits/rejected": 0.525688886642456, + "logps/chosen": -104.57971954345703, + "logps/ref_chosen": -53.325008392333984, + "logps/ref_rejected": -83.21236419677734, + "logps/rejected": -165.90138244628906, + "loss": 0.9804, + "margin_dpo/margin_mean": 31.434314727783203, + "margin_dpo/margin_std": 42.93732452392578, + "step": 349 + }, + { + "KL/chosen_KL_mean": -55.56896209716797, + "KL/mean": -75.05320739746094, + "KL/rejected_KL_mean": -94.53744506835938, + "KL/std": 44.367515563964844, + "epoch": 0.5291005291005291, + "fcm_dpo/beta": 0.02422870323061943, + "fcm_dpo/delta": -0.5494685769081116, + "fcm_dpo/margin": 38.96849060058594, + "fcm_dpo/q_t": 0.31839799880981445, + "grad_norm": 20.68731689453125, + "learning_rate": 2.698124892141971e-07, + "logits/chosen": 0.5880172252655029, + "logits/rejected": 0.504309892654419, + "logps/chosen": -117.19473266601562, + "logps/ref_chosen": -61.625770568847656, + "logps/ref_rejected": -87.63627624511719, + "logps/rejected": -182.17372131347656, + "loss": 0.8703, + "margin_dpo/margin_mean": 38.96849060058594, + "margin_dpo/margin_std": 43.68225860595703, + "step": 350 + }, + { + "KL/chosen_KL_mean": -51.41705322265625, + "KL/mean": -67.60037994384766, + "KL/rejected_KL_mean": -83.78369903564453, + "KL/std": 39.729591369628906, + "epoch": 0.5306122448979592, + "fcm_dpo/beta": 0.024021202698349953, + "fcm_dpo/delta": -0.3806014657020569, + "fcm_dpo/margin": 32.36665344238281, + "fcm_dpo/q_t": 0.3423922061920166, + "grad_norm": 16.470895767211914, + "learning_rate": 2.6849415780518357e-07, + "logits/chosen": 0.5403543710708618, + "logits/rejected": 0.4637254476547241, + "logps/chosen": -107.67338562011719, + "logps/ref_chosen": -56.2563362121582, + "logps/ref_rejected": -79.11589813232422, + "logps/rejected": -162.89959716796875, + "loss": 1.0377, + "margin_dpo/margin_mean": 32.36665344238281, + "margin_dpo/margin_std": 48.304725646972656, + "step": 351 + }, + { + "KL/chosen_KL_mean": -49.924076080322266, + "KL/mean": -65.0379638671875, + "KL/rejected_KL_mean": -80.15184783935547, + "KL/std": 40.84290313720703, + "epoch": 0.5321239606953893, + "fcm_dpo/beta": 0.02385556697845459, + "fcm_dpo/delta": -0.3236500322818756, + "fcm_dpo/margin": 30.227785110473633, + "fcm_dpo/q_t": 0.35969775915145874, + "grad_norm": 17.726348876953125, + "learning_rate": 2.6717530907482024e-07, + "logits/chosen": 0.6038858294487, + "logits/rejected": 0.5469635128974915, + "logps/chosen": -112.97602844238281, + "logps/ref_chosen": -63.05195236206055, + "logps/ref_rejected": -85.52035522460938, + "logps/rejected": -165.67221069335938, + "loss": 0.9917, + "margin_dpo/margin_mean": 30.227785110473633, + "margin_dpo/margin_std": 42.003578186035156, + "step": 352 + }, + { + "KL/chosen_KL_mean": -46.682472229003906, + "KL/mean": -63.69928741455078, + "KL/rejected_KL_mean": -80.71609497070312, + "KL/std": 41.074684143066406, + "epoch": 0.5336356764928194, + "fcm_dpo/beta": 0.023698175325989723, + "fcm_dpo/delta": -0.4096619486808777, + "fcm_dpo/margin": 34.03361892700195, + "fcm_dpo/q_t": 0.34306591749191284, + "grad_norm": 16.375343322753906, + "learning_rate": 2.658559799141411e-07, + "logits/chosen": 0.6047755479812622, + "logits/rejected": 0.6077345013618469, + "logps/chosen": -115.69165802001953, + "logps/ref_chosen": -69.00918579101562, + "logps/ref_rejected": -72.65840148925781, + "logps/rejected": -153.37449645996094, + "loss": 0.9775, + "margin_dpo/margin_mean": 34.03361892700195, + "margin_dpo/margin_std": 45.82502746582031, + "step": 353 + }, + { + "KL/chosen_KL_mean": -48.81342697143555, + "KL/mean": -66.48347473144531, + "KL/rejected_KL_mean": -84.15353393554688, + "KL/std": 39.69056701660156, + "epoch": 0.5351473922902494, + "fcm_dpo/beta": 0.023463979363441467, + "fcm_dpo/delta": -0.43324893712997437, + "fcm_dpo/margin": 35.34010314941406, + "fcm_dpo/q_t": 0.33958515524864197, + "grad_norm": 18.54259490966797, + "learning_rate": 2.6453620722761895e-07, + "logits/chosen": 0.6717275381088257, + "logits/rejected": 0.5359009504318237, + "logps/chosen": -88.60176086425781, + "logps/ref_chosen": -39.78833770751953, + "logps/ref_rejected": -69.56885528564453, + "logps/rejected": -153.72238159179688, + "loss": 0.9654, + "margin_dpo/margin_mean": 35.34010314941406, + "margin_dpo/margin_std": 46.11918640136719, + "step": 354 + }, + { + "KL/chosen_KL_mean": -54.502113342285156, + "KL/mean": -72.43010711669922, + "KL/rejected_KL_mean": -90.35810852050781, + "KL/std": 44.27423858642578, + "epoch": 0.5366591080876795, + "fcm_dpo/beta": 0.02328827604651451, + "fcm_dpo/delta": -0.4385252594947815, + "fcm_dpo/margin": 35.85599136352539, + "fcm_dpo/q_t": 0.33935868740081787, + "grad_norm": 19.201040267944336, + "learning_rate": 2.632160279321328e-07, + "logits/chosen": 0.6401563882827759, + "logits/rejected": 0.5030049681663513, + "logps/chosen": -100.75749206542969, + "logps/ref_chosen": -46.25537872314453, + "logps/ref_rejected": -78.20236206054688, + "logps/rejected": -168.56045532226562, + "loss": 0.9765, + "margin_dpo/margin_mean": 35.855987548828125, + "margin_dpo/margin_std": 48.005157470703125, + "step": 355 + }, + { + "KL/chosen_KL_mean": -48.93628692626953, + "KL/mean": -63.257347106933594, + "KL/rejected_KL_mean": -77.57839965820312, + "KL/std": 43.08306121826172, + "epoch": 0.5381708238851096, + "fcm_dpo/beta": 0.023118235170841217, + "fcm_dpo/delta": -0.2641822099685669, + "fcm_dpo/margin": 28.642108917236328, + "fcm_dpo/q_t": 0.37275728583335876, + "grad_norm": 17.23994255065918, + "learning_rate": 2.618954789559356e-07, + "logits/chosen": 0.6150566339492798, + "logits/rejected": 0.5331417322158813, + "logps/chosen": -96.84244537353516, + "logps/ref_chosen": -47.906158447265625, + "logps/ref_rejected": -74.29397583007812, + "logps/rejected": -151.87237548828125, + "loss": 1.1641, + "margin_dpo/margin_mean": 28.642108917236328, + "margin_dpo/margin_std": 54.53860092163086, + "step": 356 + }, + { + "KL/chosen_KL_mean": -58.44605255126953, + "KL/mean": -70.96112060546875, + "KL/rejected_KL_mean": -83.47620391845703, + "KL/std": 41.42731857299805, + "epoch": 0.5396825396825397, + "fcm_dpo/beta": 0.022982841357588768, + "fcm_dpo/delta": -0.26209163665771484, + "fcm_dpo/margin": 25.0301456451416, + "fcm_dpo/q_t": 0.38306644558906555, + "grad_norm": 17.207334518432617, + "learning_rate": 2.6057459723762076e-07, + "logits/chosen": 0.6086611747741699, + "logits/rejected": 0.5811838507652283, + "logps/chosen": -121.0810546875, + "logps/ref_chosen": -62.63500213623047, + "logps/ref_rejected": -65.11399841308594, + "logps/rejected": -148.5902099609375, + "loss": 1.1231, + "margin_dpo/margin_mean": 25.0301456451416, + "margin_dpo/margin_std": 42.72869110107422, + "step": 357 + }, + { + "KL/chosen_KL_mean": -54.6973991394043, + "KL/mean": -72.26055908203125, + "KL/rejected_KL_mean": -89.82373046875, + "KL/std": 42.57823181152344, + "epoch": 0.5411942554799698, + "fcm_dpo/beta": 0.022862719371914864, + "fcm_dpo/delta": -0.4061471223831177, + "fcm_dpo/margin": 35.12632369995117, + "fcm_dpo/q_t": 0.3476649522781372, + "grad_norm": 22.19621467590332, + "learning_rate": 2.5925341972508954e-07, + "logits/chosen": 0.5498001575469971, + "logits/rejected": 0.5607281923294067, + "logps/chosen": -121.90701293945312, + "logps/ref_chosen": -67.20960998535156, + "logps/ref_rejected": -69.34715270996094, + "logps/rejected": -159.17088317871094, + "loss": 0.9705, + "margin_dpo/margin_mean": 35.126319885253906, + "margin_dpo/margin_std": 46.79409408569336, + "step": 358 + }, + { + "KL/chosen_KL_mean": -59.853248596191406, + "KL/mean": -69.26499938964844, + "KL/rejected_KL_mean": -78.67675018310547, + "KL/std": 41.81439971923828, + "epoch": 0.5427059712773998, + "fcm_dpo/beta": 0.022764787077903748, + "fcm_dpo/delta": -0.14110645651817322, + "fcm_dpo/margin": 18.82351303100586, + "fcm_dpo/q_t": 0.41456860303878784, + "grad_norm": 20.174962997436523, + "learning_rate": 2.579319833745169e-07, + "logits/chosen": 0.5681227445602417, + "logits/rejected": 0.5396552085876465, + "logps/chosen": -122.3790283203125, + "logps/ref_chosen": -62.52578353881836, + "logps/ref_rejected": -76.63114929199219, + "logps/rejected": -155.30789184570312, + "loss": 1.2009, + "margin_dpo/margin_mean": 18.82351303100586, + "margin_dpo/margin_std": 40.02119064331055, + "step": 359 + }, + { + "KL/chosen_KL_mean": -58.42548751831055, + "KL/mean": -73.34075164794922, + "KL/rejected_KL_mean": -88.25601959228516, + "KL/std": 43.21331024169922, + "epoch": 0.54421768707483, + "fcm_dpo/beta": 0.022632470354437828, + "fcm_dpo/delta": -0.2770967185497284, + "fcm_dpo/margin": 29.830528259277344, + "fcm_dpo/q_t": 0.3672603964805603, + "grad_norm": 17.653297424316406, + "learning_rate": 2.5661032514931834e-07, + "logits/chosen": 0.5061342716217041, + "logits/rejected": 0.4103896915912628, + "logps/chosen": -121.9132080078125, + "logps/ref_chosen": -63.48772048950195, + "logps/ref_rejected": -90.6891098022461, + "logps/rejected": -178.94512939453125, + "loss": 1.0149, + "margin_dpo/margin_mean": 29.830530166625977, + "margin_dpo/margin_std": 42.919334411621094, + "step": 360 + }, + { + "KL/chosen_KL_mean": -55.86784362792969, + "KL/mean": -73.78764343261719, + "KL/rejected_KL_mean": -91.70742797851562, + "KL/std": 44.953086853027344, + "epoch": 0.54572940287226, + "fcm_dpo/beta": 0.022471774369478226, + "fcm_dpo/delta": -0.408625066280365, + "fcm_dpo/margin": 35.83958435058594, + "fcm_dpo/q_t": 0.3382977843284607, + "grad_norm": 16.23537254333496, + "learning_rate": 2.552884820191154e-07, + "logits/chosen": 0.6556925773620605, + "logits/rejected": 0.6028813123703003, + "logps/chosen": -113.78498840332031, + "logps/ref_chosen": -57.917144775390625, + "logps/ref_rejected": -72.39089965820312, + "logps/rejected": -164.09832763671875, + "loss": 0.9329, + "margin_dpo/margin_mean": 35.83958435058594, + "margin_dpo/margin_std": 43.46430206298828, + "step": 361 + }, + { + "KL/chosen_KL_mean": -54.637840270996094, + "KL/mean": -73.20539855957031, + "KL/rejected_KL_mean": -91.77296447753906, + "KL/std": 44.35628128051758, + "epoch": 0.54724111866969, + "fcm_dpo/beta": 0.022302858531475067, + "fcm_dpo/delta": -0.4315636456012726, + "fcm_dpo/margin": 37.13511657714844, + "fcm_dpo/q_t": 0.34225910902023315, + "grad_norm": 17.085247039794922, + "learning_rate": 2.53966490958702e-07, + "logits/chosen": 0.6554888486862183, + "logits/rejected": 0.5415962934494019, + "logps/chosen": -118.08131408691406, + "logps/ref_chosen": -63.4434700012207, + "logps/ref_rejected": -103.45516967773438, + "logps/rejected": -195.22813415527344, + "loss": 0.9515, + "margin_dpo/margin_mean": 37.1351203918457, + "margin_dpo/margin_std": 47.92539978027344, + "step": 362 + }, + { + "KL/chosen_KL_mean": -59.362972259521484, + "KL/mean": -77.52494812011719, + "KL/rejected_KL_mean": -95.68692016601562, + "KL/std": 42.81013107299805, + "epoch": 0.5487528344671202, + "fcm_dpo/beta": 0.022091105580329895, + "fcm_dpo/delta": -0.40578335523605347, + "fcm_dpo/margin": 36.32394790649414, + "fcm_dpo/q_t": 0.34126996994018555, + "grad_norm": 16.03322410583496, + "learning_rate": 2.526443889470099e-07, + "logits/chosen": 0.6519886255264282, + "logits/rejected": 0.5134670734405518, + "logps/chosen": -108.01480102539062, + "logps/ref_chosen": -48.65182876586914, + "logps/ref_rejected": -88.65904235839844, + "logps/rejected": -184.34597778320312, + "loss": 0.9359, + "margin_dpo/margin_mean": 36.323951721191406, + "margin_dpo/margin_std": 45.14124298095703, + "step": 363 + }, + { + "KL/chosen_KL_mean": -53.51654815673828, + "KL/mean": -72.5213623046875, + "KL/rejected_KL_mean": -91.52616119384766, + "KL/std": 46.348045349121094, + "epoch": 0.5502645502645502, + "fcm_dpo/beta": 0.021906405687332153, + "fcm_dpo/delta": -0.4362999200820923, + "fcm_dpo/margin": 38.009620666503906, + "fcm_dpo/q_t": 0.34411001205444336, + "grad_norm": 16.67765998840332, + "learning_rate": 2.513222129660744e-07, + "logits/chosen": 0.5093514919281006, + "logits/rejected": 0.41392073035240173, + "logps/chosen": -111.38761901855469, + "logps/ref_chosen": -57.87107467651367, + "logps/ref_rejected": -80.95503234863281, + "logps/rejected": -172.481201171875, + "loss": 0.9938, + "margin_dpo/margin_mean": 38.009620666503906, + "margin_dpo/margin_std": 53.821746826171875, + "step": 364 + }, + { + "KL/chosen_KL_mean": -47.06479263305664, + "KL/mean": -64.76912689208984, + "KL/rejected_KL_mean": -82.47346496582031, + "KL/std": 47.71750259399414, + "epoch": 0.5517762660619804, + "fcm_dpo/beta": 0.02173316478729248, + "fcm_dpo/delta": -0.3724898099899292, + "fcm_dpo/margin": 35.40867614746094, + "fcm_dpo/q_t": 0.34288179874420166, + "grad_norm": 16.262985229492188, + "learning_rate": 2.5e-07, + "logits/chosen": 0.6128625273704529, + "logits/rejected": 0.6047656536102295, + "logps/chosen": -112.00697326660156, + "logps/ref_chosen": -64.94217681884766, + "logps/ref_rejected": -74.8599853515625, + "logps/rejected": -157.3334503173828, + "loss": 0.9177, + "margin_dpo/margin_mean": 35.40867614746094, + "margin_dpo/margin_std": 40.5378532409668, + "step": 365 + }, + { + "KL/chosen_KL_mean": -44.4941520690918, + "KL/mean": -60.347145080566406, + "KL/rejected_KL_mean": -76.20013427734375, + "KL/std": 40.4246826171875, + "epoch": 0.5532879818594104, + "fcm_dpo/beta": 0.02162015810608864, + "fcm_dpo/delta": -0.28735658526420593, + "fcm_dpo/margin": 31.705989837646484, + "fcm_dpo/q_t": 0.36733222007751465, + "grad_norm": 16.229087829589844, + "learning_rate": 2.486777870339255e-07, + "logits/chosen": 0.5436862707138062, + "logits/rejected": 0.5264623761177063, + "logps/chosen": -99.6601333618164, + "logps/ref_chosen": -55.16598129272461, + "logps/ref_rejected": -65.26121520996094, + "logps/rejected": -141.4613494873047, + "loss": 1.0495, + "margin_dpo/margin_mean": 31.705989837646484, + "margin_dpo/margin_std": 48.71324157714844, + "step": 366 + }, + { + "KL/chosen_KL_mean": -54.5137825012207, + "KL/mean": -68.82611083984375, + "KL/rejected_KL_mean": -83.138427734375, + "KL/std": 42.735687255859375, + "epoch": 0.5547996976568406, + "fcm_dpo/beta": 0.021497860550880432, + "fcm_dpo/delta": -0.2167588174343109, + "fcm_dpo/margin": 28.624645233154297, + "fcm_dpo/q_t": 0.3705470860004425, + "grad_norm": 17.30897331237793, + "learning_rate": 2.4735561105299014e-07, + "logits/chosen": 0.5858331918716431, + "logits/rejected": 0.4766313433647156, + "logps/chosen": -110.52424621582031, + "logps/ref_chosen": -56.01046371459961, + "logps/ref_rejected": -77.31010437011719, + "logps/rejected": -160.44854736328125, + "loss": 1.0542, + "margin_dpo/margin_mean": 28.624645233154297, + "margin_dpo/margin_std": 43.82920837402344, + "step": 367 + }, + { + "KL/chosen_KL_mean": -60.24021911621094, + "KL/mean": -74.1727294921875, + "KL/rejected_KL_mean": -88.1052474975586, + "KL/std": 43.06662368774414, + "epoch": 0.5563114134542706, + "fcm_dpo/beta": 0.02141546830534935, + "fcm_dpo/delta": -0.1979246586561203, + "fcm_dpo/margin": 27.86503028869629, + "fcm_dpo/q_t": 0.3760097026824951, + "grad_norm": 15.950251579284668, + "learning_rate": 2.46033509041298e-07, + "logits/chosen": 0.4400317668914795, + "logits/rejected": 0.4355320930480957, + "logps/chosen": -135.06948852539062, + "logps/ref_chosen": -74.82927703857422, + "logps/ref_rejected": -76.11680603027344, + "logps/rejected": -164.2220458984375, + "loss": 1.0819, + "margin_dpo/margin_mean": 27.865032196044922, + "margin_dpo/margin_std": 45.282691955566406, + "step": 368 + }, + { + "KL/chosen_KL_mean": -56.79549789428711, + "KL/mean": -70.24311828613281, + "KL/rejected_KL_mean": -83.69075775146484, + "KL/std": 43.80937576293945, + "epoch": 0.5578231292517006, + "fcm_dpo/beta": 0.021351777017116547, + "fcm_dpo/delta": -0.17523756623268127, + "fcm_dpo/margin": 26.89525604248047, + "fcm_dpo/q_t": 0.38354814052581787, + "grad_norm": 16.345746994018555, + "learning_rate": 2.447115179808846e-07, + "logits/chosen": 0.590995728969574, + "logits/rejected": 0.5369537472724915, + "logps/chosen": -115.12171936035156, + "logps/ref_chosen": -58.32621765136719, + "logps/ref_rejected": -80.92183685302734, + "logps/rejected": -164.6125946044922, + "loss": 1.0846, + "margin_dpo/margin_mean": 26.89525604248047, + "margin_dpo/margin_std": 44.078704833984375, + "step": 369 + }, + { + "KL/chosen_KL_mean": -54.44055938720703, + "KL/mean": -72.711181640625, + "KL/rejected_KL_mean": -90.9817886352539, + "KL/std": 46.546295166015625, + "epoch": 0.5593348450491308, + "fcm_dpo/beta": 0.02119690179824829, + "fcm_dpo/delta": -0.3775292634963989, + "fcm_dpo/margin": 36.541236877441406, + "fcm_dpo/q_t": 0.3481212854385376, + "grad_norm": 15.878318786621094, + "learning_rate": 2.4338967485068164e-07, + "logits/chosen": 0.6807664632797241, + "logits/rejected": 0.6093059778213501, + "logps/chosen": -107.32427978515625, + "logps/ref_chosen": -52.88372039794922, + "logps/ref_rejected": -79.43692016601562, + "logps/rejected": -170.418701171875, + "loss": 1.0158, + "margin_dpo/margin_mean": 36.541236877441406, + "margin_dpo/margin_std": 53.44210433959961, + "step": 370 + }, + { + "KL/chosen_KL_mean": -52.433837890625, + "KL/mean": -69.87709045410156, + "KL/rejected_KL_mean": -87.32034301757812, + "KL/std": 45.355682373046875, + "epoch": 0.5608465608465608, + "fcm_dpo/beta": 0.02106933295726776, + "fcm_dpo/delta": -0.33737486600875854, + "fcm_dpo/margin": 34.88650894165039, + "fcm_dpo/q_t": 0.3548740744590759, + "grad_norm": 15.176125526428223, + "learning_rate": 2.420680166254831e-07, + "logits/chosen": 0.7046289443969727, + "logits/rejected": 0.6696420907974243, + "logps/chosen": -101.65805053710938, + "logps/ref_chosen": -49.224212646484375, + "logps/ref_rejected": -63.348472595214844, + "logps/rejected": -150.6688232421875, + "loss": 0.9787, + "margin_dpo/margin_mean": 34.88650894165039, + "margin_dpo/margin_std": 45.74319839477539, + "step": 371 + }, + { + "KL/chosen_KL_mean": -58.290245056152344, + "KL/mean": -68.91067504882812, + "KL/rejected_KL_mean": -79.53108978271484, + "KL/std": 43.23418045043945, + "epoch": 0.562358276643991, + "fcm_dpo/beta": 0.02102331444621086, + "fcm_dpo/delta": -0.04691962152719498, + "fcm_dpo/margin": 21.240842819213867, + "fcm_dpo/q_t": 0.4115324020385742, + "grad_norm": 19.809812545776367, + "learning_rate": 2.4074658027491044e-07, + "logits/chosen": 0.6246475577354431, + "logits/rejected": 0.5273189544677734, + "logps/chosen": -110.55979919433594, + "logps/ref_chosen": -52.269554138183594, + "logps/ref_rejected": -72.99522399902344, + "logps/rejected": -152.52630615234375, + "loss": 1.2682, + "margin_dpo/margin_mean": 21.240842819213867, + "margin_dpo/margin_std": 53.36918640136719, + "step": 372 + }, + { + "KL/chosen_KL_mean": -64.22086334228516, + "KL/mean": -77.72557830810547, + "KL/rejected_KL_mean": -91.23028564453125, + "KL/std": 44.7161750793457, + "epoch": 0.563869992441421, + "fcm_dpo/beta": 0.020946351811289787, + "fcm_dpo/delta": -0.16668005287647247, + "fcm_dpo/margin": 27.009418487548828, + "fcm_dpo/q_t": 0.3908725380897522, + "grad_norm": 22.030263900756836, + "learning_rate": 2.394254027623792e-07, + "logits/chosen": 0.6170238256454468, + "logits/rejected": 0.5430665016174316, + "logps/chosen": -125.3338623046875, + "logps/ref_chosen": -61.112998962402344, + "logps/ref_rejected": -76.24851989746094, + "logps/rejected": -167.4788055419922, + "loss": 1.1749, + "margin_dpo/margin_mean": 27.009418487548828, + "margin_dpo/margin_std": 54.32630920410156, + "step": 373 + }, + { + "KL/chosen_KL_mean": -55.723060607910156, + "KL/mean": -77.4720230102539, + "KL/rejected_KL_mean": -99.22099304199219, + "KL/std": 44.75810623168945, + "epoch": 0.5653817082388511, + "fcm_dpo/beta": 0.020764198154211044, + "fcm_dpo/delta": -0.5079047083854675, + "fcm_dpo/margin": 43.4979248046875, + "fcm_dpo/q_t": 0.31678080558776855, + "grad_norm": 23.113033294677734, + "learning_rate": 2.381045210440644e-07, + "logits/chosen": 0.5161197185516357, + "logits/rejected": 0.5129517912864685, + "logps/chosen": -128.39227294921875, + "logps/ref_chosen": -72.66920471191406, + "logps/ref_rejected": -76.83158874511719, + "logps/rejected": -176.05258178710938, + "loss": 0.8481, + "margin_dpo/margin_mean": 43.4979248046875, + "margin_dpo/margin_std": 44.506309509277344, + "step": 374 + }, + { + "KL/chosen_KL_mean": -54.631805419921875, + "KL/mean": -69.17517852783203, + "KL/rejected_KL_mean": -83.71855163574219, + "KL/std": 46.84748840332031, + "epoch": 0.5668934240362812, + "fcm_dpo/beta": 0.020642247051000595, + "fcm_dpo/delta": -0.20178920030593872, + "fcm_dpo/margin": 29.086748123168945, + "fcm_dpo/q_t": 0.37976503372192383, + "grad_norm": 18.89912986755371, + "learning_rate": 2.3678397206786715e-07, + "logits/chosen": 0.6185399293899536, + "logits/rejected": 0.5577261447906494, + "logps/chosen": -112.31510925292969, + "logps/ref_chosen": -57.68330383300781, + "logps/ref_rejected": -79.34097290039062, + "logps/rejected": -163.0595245361328, + "loss": 1.0866, + "margin_dpo/margin_mean": 29.086748123168945, + "margin_dpo/margin_std": 48.184165954589844, + "step": 375 + }, + { + "KL/chosen_KL_mean": -59.527374267578125, + "KL/mean": -79.89494323730469, + "KL/rejected_KL_mean": -100.26252746582031, + "KL/std": 48.166786193847656, + "epoch": 0.5684051398337112, + "fcm_dpo/beta": 0.02049219235777855, + "fcm_dpo/delta": -0.4385319948196411, + "fcm_dpo/margin": 40.73514938354492, + "fcm_dpo/q_t": 0.34296852350234985, + "grad_norm": 16.208314895629883, + "learning_rate": 2.3546379277238103e-07, + "logits/chosen": 0.6611849069595337, + "logits/rejected": 0.5875750780105591, + "logps/chosen": -111.20144653320312, + "logps/ref_chosen": -51.674072265625, + "logps/ref_rejected": -75.69713592529297, + "logps/rejected": -175.95965576171875, + "loss": 0.969, + "margin_dpo/margin_mean": 40.73515319824219, + "margin_dpo/margin_std": 54.91881561279297, + "step": 376 + }, + { + "KL/chosen_KL_mean": -59.38386154174805, + "KL/mean": -74.05152130126953, + "KL/rejected_KL_mean": -88.71918487548828, + "KL/std": 44.05195617675781, + "epoch": 0.5699168556311414, + "fcm_dpo/beta": 0.020404577255249023, + "fcm_dpo/delta": -0.19972655177116394, + "fcm_dpo/margin": 29.335323333740234, + "fcm_dpo/q_t": 0.37659668922424316, + "grad_norm": 17.41449737548828, + "learning_rate": 2.3414402008585886e-07, + "logits/chosen": 0.6146732568740845, + "logits/rejected": 0.5907267332077026, + "logps/chosen": -105.56239318847656, + "logps/ref_chosen": -46.17853546142578, + "logps/ref_rejected": -57.756500244140625, + "logps/rejected": -146.47567749023438, + "loss": 1.0686, + "margin_dpo/margin_mean": 29.335323333740234, + "margin_dpo/margin_std": 45.85279083251953, + "step": 377 + }, + { + "KL/chosen_KL_mean": -57.24024963378906, + "KL/mean": -70.55381774902344, + "KL/rejected_KL_mean": -83.86738586425781, + "KL/std": 45.73373794555664, + "epoch": 0.5714285714285714, + "fcm_dpo/beta": 0.020340571179986, + "fcm_dpo/delta": -0.14237552881240845, + "fcm_dpo/margin": 26.627132415771484, + "fcm_dpo/q_t": 0.3895426392555237, + "grad_norm": 16.96466827392578, + "learning_rate": 2.3282469092517977e-07, + "logits/chosen": 0.6600319147109985, + "logits/rejected": 0.6091748476028442, + "logps/chosen": -116.4591293334961, + "logps/ref_chosen": -59.21887969970703, + "logps/ref_rejected": -71.24818420410156, + "logps/rejected": -155.11557006835938, + "loss": 1.1132, + "margin_dpo/margin_mean": 26.62712860107422, + "margin_dpo/margin_std": 46.22966384887695, + "step": 378 + }, + { + "KL/chosen_KL_mean": -54.50444793701172, + "KL/mean": -72.15367126464844, + "KL/rejected_KL_mean": -89.80290985107422, + "KL/std": 45.22607421875, + "epoch": 0.5729402872260015, + "fcm_dpo/beta": 0.02021736651659012, + "fcm_dpo/delta": -0.3158871829509735, + "fcm_dpo/margin": 35.298465728759766, + "fcm_dpo/q_t": 0.3585625886917114, + "grad_norm": 16.37108039855957, + "learning_rate": 2.3150584219481643e-07, + "logits/chosen": 0.6335070133209229, + "logits/rejected": 0.5568963289260864, + "logps/chosen": -130.82102966308594, + "logps/ref_chosen": -76.31658935546875, + "logps/ref_rejected": -104.26200103759766, + "logps/rejected": -194.06491088867188, + "loss": 0.9806, + "margin_dpo/margin_mean": 35.29846954345703, + "margin_dpo/margin_std": 47.386653900146484, + "step": 379 + }, + { + "KL/chosen_KL_mean": -49.015689849853516, + "KL/mean": -69.79548645019531, + "KL/rejected_KL_mean": -90.57528686523438, + "KL/std": 43.08797073364258, + "epoch": 0.5744520030234316, + "fcm_dpo/beta": 0.020055226981639862, + "fcm_dpo/delta": -0.43708565831184387, + "fcm_dpo/margin": 41.55960464477539, + "fcm_dpo/q_t": 0.3334227502346039, + "grad_norm": 15.668764114379883, + "learning_rate": 2.3018751078580283e-07, + "logits/chosen": 0.6232728958129883, + "logits/rejected": 0.5814328193664551, + "logps/chosen": -110.29885864257812, + "logps/ref_chosen": -61.283164978027344, + "logps/ref_rejected": -72.38892364501953, + "logps/rejected": -162.96420288085938, + "loss": 0.9319, + "margin_dpo/margin_mean": 41.55960464477539, + "margin_dpo/margin_std": 50.423133850097656, + "step": 380 + }, + { + "KL/chosen_KL_mean": -60.613739013671875, + "KL/mean": -69.33210754394531, + "KL/rejected_KL_mean": -78.05046081542969, + "KL/std": 44.04750061035156, + "epoch": 0.5759637188208617, + "fcm_dpo/beta": 0.01999620907008648, + "fcm_dpo/delta": -0.0641731470823288, + "fcm_dpo/margin": 17.436717987060547, + "fcm_dpo/q_t": 0.4304971396923065, + "grad_norm": 16.772628784179688, + "learning_rate": 2.288697335747027e-07, + "logits/chosen": 0.5754144191741943, + "logits/rejected": 0.5504010915756226, + "logps/chosen": -118.82772827148438, + "logps/ref_chosen": -58.2139892578125, + "logps/ref_rejected": -60.78669357299805, + "logps/rejected": -138.837158203125, + "loss": 1.2794, + "margin_dpo/margin_mean": 17.436717987060547, + "margin_dpo/margin_std": 48.51057052612305, + "step": 381 + }, + { + "KL/chosen_KL_mean": -61.97425842285156, + "KL/mean": -75.79351043701172, + "KL/rejected_KL_mean": -89.61276245117188, + "KL/std": 46.382591247558594, + "epoch": 0.5774754346182918, + "fcm_dpo/beta": 0.019951093941926956, + "fcm_dpo/delta": -0.1522451788187027, + "fcm_dpo/margin": 27.638500213623047, + "fcm_dpo/q_t": 0.3856785297393799, + "grad_norm": 16.234302520751953, + "learning_rate": 2.2755254742257706e-07, + "logits/chosen": 0.606398344039917, + "logits/rejected": 0.5514424443244934, + "logps/chosen": -123.7995834350586, + "logps/ref_chosen": -61.82532501220703, + "logps/ref_rejected": -83.0452880859375, + "logps/rejected": -172.65805053710938, + "loss": 1.0567, + "margin_dpo/margin_mean": 27.638500213623047, + "margin_dpo/margin_std": 41.325599670410156, + "step": 382 + }, + { + "KL/chosen_KL_mean": -57.607337951660156, + "KL/mean": -74.9368896484375, + "KL/rejected_KL_mean": -92.26644897460938, + "KL/std": 45.43861389160156, + "epoch": 0.5789871504157218, + "fcm_dpo/beta": 0.019824445247650146, + "fcm_dpo/delta": -0.28933292627334595, + "fcm_dpo/margin": 34.659122467041016, + "fcm_dpo/q_t": 0.3670164942741394, + "grad_norm": 19.112213134765625, + "learning_rate": 2.2623598917395436e-07, + "logits/chosen": 0.49152398109436035, + "logits/rejected": 0.5194563865661621, + "logps/chosen": -138.17059326171875, + "logps/ref_chosen": -80.56326293945312, + "logps/ref_rejected": -74.62922668457031, + "logps/rejected": -166.8956756591797, + "loss": 1.0373, + "margin_dpo/margin_mean": 34.659122467041016, + "margin_dpo/margin_std": 52.22249984741211, + "step": 383 + }, + { + "KL/chosen_KL_mean": -57.48443603515625, + "KL/mean": -74.17256164550781, + "KL/rejected_KL_mean": -90.86068725585938, + "KL/std": 45.036895751953125, + "epoch": 0.5804988662131519, + "fcm_dpo/beta": 0.019735019654035568, + "fcm_dpo/delta": -0.2603476345539093, + "fcm_dpo/margin": 33.376251220703125, + "fcm_dpo/q_t": 0.3617045283317566, + "grad_norm": 15.098806381225586, + "learning_rate": 2.2492009565579875e-07, + "logits/chosen": 0.6107144355773926, + "logits/rejected": 0.5639744400978088, + "logps/chosen": -122.95957946777344, + "logps/ref_chosen": -65.47514343261719, + "logps/ref_rejected": -79.67378234863281, + "logps/rejected": -170.5344696044922, + "loss": 0.9865, + "margin_dpo/margin_mean": 33.376251220703125, + "margin_dpo/margin_std": 43.13768768310547, + "step": 384 + }, + { + "KL/chosen_KL_mean": -58.00761413574219, + "KL/mean": -77.44103240966797, + "KL/rejected_KL_mean": -96.87445068359375, + "KL/std": 44.22943878173828, + "epoch": 0.582010582010582, + "fcm_dpo/beta": 0.019603757187724113, + "fcm_dpo/delta": -0.36461904644966125, + "fcm_dpo/margin": 38.86683654785156, + "fcm_dpo/q_t": 0.34423407912254333, + "grad_norm": 14.788227081298828, + "learning_rate": 2.2360490367648084e-07, + "logits/chosen": 0.5295755863189697, + "logits/rejected": 0.487786203622818, + "logps/chosen": -124.06413269042969, + "logps/ref_chosen": -66.0565185546875, + "logps/ref_rejected": -86.68023681640625, + "logps/rejected": -183.5546875, + "loss": 0.9264, + "margin_dpo/margin_mean": 38.86683654785156, + "margin_dpo/margin_std": 44.79783630371094, + "step": 385 + }, + { + "KL/chosen_KL_mean": -66.8941650390625, + "KL/mean": -80.02250671386719, + "KL/rejected_KL_mean": -93.15084838867188, + "KL/std": 47.047054290771484, + "epoch": 0.5835222978080121, + "fcm_dpo/beta": 0.01952105388045311, + "fcm_dpo/delta": -0.11321018636226654, + "fcm_dpo/margin": 26.256698608398438, + "fcm_dpo/q_t": 0.3910138010978699, + "grad_norm": 17.788394927978516, + "learning_rate": 2.2229045002474724e-07, + "logits/chosen": 0.5282996892929077, + "logits/rejected": 0.46648108959198, + "logps/chosen": -142.517822265625, + "logps/ref_chosen": -75.6236572265625, + "logps/ref_rejected": -92.62330627441406, + "logps/rejected": -185.774169921875, + "loss": 1.1063, + "margin_dpo/margin_mean": 26.256698608398438, + "margin_dpo/margin_std": 44.46893310546875, + "step": 386 + }, + { + "KL/chosen_KL_mean": -58.502342224121094, + "KL/mean": -76.95713806152344, + "KL/rejected_KL_mean": -95.41192626953125, + "KL/std": 43.727317810058594, + "epoch": 0.5850340136054422, + "fcm_dpo/beta": 0.01942615769803524, + "fcm_dpo/delta": -0.31927189230918884, + "fcm_dpo/margin": 36.90958786010742, + "fcm_dpo/q_t": 0.3540007472038269, + "grad_norm": 16.19090461730957, + "learning_rate": 2.209767714686924e-07, + "logits/chosen": 0.6206231117248535, + "logits/rejected": 0.5125142335891724, + "logps/chosen": -105.72404479980469, + "logps/ref_chosen": -47.22170639038086, + "logps/ref_rejected": -87.338134765625, + "logps/rejected": -182.75006103515625, + "loss": 0.9414, + "margin_dpo/margin_mean": 36.909584045410156, + "margin_dpo/margin_std": 44.024436950683594, + "step": 387 + }, + { + "KL/chosen_KL_mean": -58.7374153137207, + "KL/mean": -71.86933898925781, + "KL/rejected_KL_mean": -85.00127410888672, + "KL/std": 44.9259033203125, + "epoch": 0.5865457294028723, + "fcm_dpo/beta": 0.019349249079823494, + "fcm_dpo/delta": -0.10888849198818207, + "fcm_dpo/margin": 26.26386260986328, + "fcm_dpo/q_t": 0.4000864624977112, + "grad_norm": 16.325885772705078, + "learning_rate": 2.1966390475472954e-07, + "logits/chosen": 0.5971830487251282, + "logits/rejected": 0.5869323015213013, + "logps/chosen": -133.31689453125, + "logps/ref_chosen": -74.5794677734375, + "logps/ref_rejected": -79.92558288574219, + "logps/rejected": -164.92684936523438, + "loss": 1.1504, + "margin_dpo/margin_mean": 26.26386260986328, + "margin_dpo/margin_std": 50.83785629272461, + "step": 388 + }, + { + "KL/chosen_KL_mean": -60.47760009765625, + "KL/mean": -78.75874328613281, + "KL/rejected_KL_mean": -97.03990936279297, + "KL/std": 46.49164581298828, + "epoch": 0.5880574452003023, + "fcm_dpo/beta": 0.019267920404672623, + "fcm_dpo/delta": -0.3065676987171173, + "fcm_dpo/margin": 36.56230926513672, + "fcm_dpo/q_t": 0.3575364351272583, + "grad_norm": 40.1843376159668, + "learning_rate": 2.1835188660656265e-07, + "logits/chosen": 0.5708217620849609, + "logits/rejected": 0.5339006781578064, + "logps/chosen": -122.10195922851562, + "logps/ref_chosen": -61.624366760253906, + "logps/ref_rejected": -76.50978088378906, + "logps/rejected": -173.5496826171875, + "loss": 0.9991, + "margin_dpo/margin_mean": 36.56230926513672, + "margin_dpo/margin_std": 50.190765380859375, + "step": 389 + }, + { + "KL/chosen_KL_mean": -53.861541748046875, + "KL/mean": -69.45010375976562, + "KL/rejected_KL_mean": -85.03866577148438, + "KL/std": 44.36452865600586, + "epoch": 0.5895691609977324, + "fcm_dpo/beta": 0.019171155989170074, + "fcm_dpo/delta": -0.1989448368549347, + "fcm_dpo/margin": 31.1771297454834, + "fcm_dpo/q_t": 0.375097393989563, + "grad_norm": 14.590592384338379, + "learning_rate": 2.170407537241599e-07, + "logits/chosen": 0.7012438178062439, + "logits/rejected": 0.6268711090087891, + "logps/chosen": -99.7333984375, + "logps/ref_chosen": -45.871864318847656, + "logps/ref_rejected": -61.305999755859375, + "logps/rejected": -146.34466552734375, + "loss": 1.0318, + "margin_dpo/margin_mean": 31.1771297454834, + "margin_dpo/margin_std": 44.34825134277344, + "step": 390 + }, + { + "KL/chosen_KL_mean": -56.79998779296875, + "KL/mean": -75.73416137695312, + "KL/rejected_KL_mean": -94.66831970214844, + "KL/std": 46.340354919433594, + "epoch": 0.5910808767951625, + "fcm_dpo/beta": 0.019059427082538605, + "fcm_dpo/delta": -0.3241666555404663, + "fcm_dpo/margin": 37.86833190917969, + "fcm_dpo/q_t": 0.3561837077140808, + "grad_norm": 15.514339447021484, + "learning_rate": 2.1573054278272636e-07, + "logits/chosen": 0.6067829132080078, + "logits/rejected": 0.5369429588317871, + "logps/chosen": -114.98699951171875, + "logps/ref_chosen": -58.18701171875, + "logps/ref_rejected": -83.63442993164062, + "logps/rejected": -178.30274963378906, + "loss": 1.0151, + "margin_dpo/margin_mean": 37.86833190917969, + "margin_dpo/margin_std": 54.56836700439453, + "step": 391 + }, + { + "KL/chosen_KL_mean": -49.55797576904297, + "KL/mean": -70.13993835449219, + "KL/rejected_KL_mean": -90.72189331054688, + "KL/std": 47.72187805175781, + "epoch": 0.5925925925925926, + "fcm_dpo/beta": 0.018941069021821022, + "fcm_dpo/delta": -0.3824905455112457, + "fcm_dpo/margin": 41.16392517089844, + "fcm_dpo/q_t": 0.3502754271030426, + "grad_norm": 14.619775772094727, + "learning_rate": 2.1442129043167873e-07, + "logits/chosen": 0.6753349304199219, + "logits/rejected": 0.6127820014953613, + "logps/chosen": -119.30250549316406, + "logps/ref_chosen": -69.7445297241211, + "logps/ref_rejected": -94.05877685546875, + "logps/rejected": -184.78067016601562, + "loss": 0.9875, + "margin_dpo/margin_mean": 41.16392517089844, + "margin_dpo/margin_std": 56.761844635009766, + "step": 392 + }, + { + "KL/chosen_KL_mean": -58.8673095703125, + "KL/mean": -79.4952621459961, + "KL/rejected_KL_mean": -100.12322235107422, + "KL/std": 46.41615295410156, + "epoch": 0.5941043083900227, + "fcm_dpo/beta": 0.018769651651382446, + "fcm_dpo/delta": -0.3775806427001953, + "fcm_dpo/margin": 41.25590133666992, + "fcm_dpo/q_t": 0.3427024781703949, + "grad_norm": 14.660417556762695, + "learning_rate": 2.131130332936195e-07, + "logits/chosen": 0.6496384143829346, + "logits/rejected": 0.6080884337425232, + "logps/chosen": -111.20220947265625, + "logps/ref_chosen": -52.33489990234375, + "logps/ref_rejected": -74.33809661865234, + "logps/rejected": -174.46131896972656, + "loss": 0.919, + "margin_dpo/margin_mean": 41.255897521972656, + "margin_dpo/margin_std": 46.9287109375, + "step": 393 + }, + { + "KL/chosen_KL_mean": -56.38057327270508, + "KL/mean": -72.58210754394531, + "KL/rejected_KL_mean": -88.78364562988281, + "KL/std": 41.60865783691406, + "epoch": 0.5956160241874527, + "fcm_dpo/beta": 0.018695756793022156, + "fcm_dpo/delta": -0.20700164139270782, + "fcm_dpo/margin": 32.4030647277832, + "fcm_dpo/q_t": 0.36907535791397095, + "grad_norm": 18.731632232666016, + "learning_rate": 2.1180580796331323e-07, + "logits/chosen": 0.6449071764945984, + "logits/rejected": 0.6124973893165588, + "logps/chosen": -117.05670928955078, + "logps/ref_chosen": -60.6761360168457, + "logps/ref_rejected": -71.36074829101562, + "logps/rejected": -160.14439392089844, + "loss": 1.0024, + "margin_dpo/margin_mean": 32.4030647277832, + "margin_dpo/margin_std": 41.15106964111328, + "step": 394 + }, + { + "KL/chosen_KL_mean": -59.952903747558594, + "KL/mean": -74.57882690429688, + "KL/rejected_KL_mean": -89.20475769042969, + "KL/std": 45.059722900390625, + "epoch": 0.5971277399848829, + "fcm_dpo/beta": 0.018637455999851227, + "fcm_dpo/delta": -0.14595665037631989, + "fcm_dpo/margin": 29.25185775756836, + "fcm_dpo/q_t": 0.3881601095199585, + "grad_norm": 18.276641845703125, + "learning_rate": 2.104996510066625e-07, + "logits/chosen": 0.613265872001648, + "logits/rejected": 0.5151522159576416, + "logps/chosen": -110.5572280883789, + "logps/ref_chosen": -50.60432434082031, + "logps/ref_rejected": -77.08731079101562, + "logps/rejected": -166.2920684814453, + "loss": 1.0692, + "margin_dpo/margin_mean": 29.25185775756836, + "margin_dpo/margin_std": 45.21484375, + "step": 395 + }, + { + "KL/chosen_KL_mean": -57.72175598144531, + "KL/mean": -75.82484436035156, + "KL/rejected_KL_mean": -93.92794799804688, + "KL/std": 51.82525634765625, + "epoch": 0.5986394557823129, + "fcm_dpo/beta": 0.01851041242480278, + "fcm_dpo/delta": -0.2728464603424072, + "fcm_dpo/margin": 36.2061882019043, + "fcm_dpo/q_t": 0.3619036376476288, + "grad_norm": 14.760275840759277, + "learning_rate": 2.0919459895968517e-07, + "logits/chosen": 0.6474554538726807, + "logits/rejected": 0.5445578098297119, + "logps/chosen": -109.08136749267578, + "logps/ref_chosen": -51.35961151123047, + "logps/ref_rejected": -79.89360046386719, + "logps/rejected": -173.821533203125, + "loss": 0.9856, + "margin_dpo/margin_mean": 36.2061882019043, + "margin_dpo/margin_std": 44.74341583251953, + "step": 396 + }, + { + "KL/chosen_KL_mean": -66.48785400390625, + "KL/mean": -75.5160903930664, + "KL/rejected_KL_mean": -84.54431915283203, + "KL/std": 47.21852111816406, + "epoch": 0.600151171579743, + "fcm_dpo/beta": 0.018537987023591995, + "fcm_dpo/delta": 0.06542832404375076, + "fcm_dpo/margin": 18.056461334228516, + "fcm_dpo/q_t": 0.4267409145832062, + "grad_norm": 18.760759353637695, + "learning_rate": 2.078906883274924e-07, + "logits/chosen": 0.5328976511955261, + "logits/rejected": 0.48425590991973877, + "logps/chosen": -132.94407653808594, + "logps/ref_chosen": -66.45622253417969, + "logps/ref_rejected": -85.74736785888672, + "logps/rejected": -170.29168701171875, + "loss": 1.3089, + "margin_dpo/margin_mean": 18.05646324157715, + "margin_dpo/margin_std": 54.185020446777344, + "step": 397 + }, + { + "KL/chosen_KL_mean": -56.39061737060547, + "KL/mean": -77.30268859863281, + "KL/rejected_KL_mean": -98.21475219726562, + "KL/std": 49.63849639892578, + "epoch": 0.6016628873771731, + "fcm_dpo/beta": 0.01842420920729637, + "fcm_dpo/delta": -0.3736782670021057, + "fcm_dpo/margin": 41.824119567871094, + "fcm_dpo/q_t": 0.346784383058548, + "grad_norm": 14.112161636352539, + "learning_rate": 2.065879555832674e-07, + "logits/chosen": 0.5939739942550659, + "logits/rejected": 0.5238075852394104, + "logps/chosen": -105.63485717773438, + "logps/ref_chosen": -49.244239807128906, + "logps/ref_rejected": -75.18949127197266, + "logps/rejected": -173.40423583984375, + "loss": 0.9246, + "margin_dpo/margin_mean": 41.824119567871094, + "margin_dpo/margin_std": 49.69524383544922, + "step": 398 + }, + { + "KL/chosen_KL_mean": -66.15803527832031, + "KL/mean": -89.28106689453125, + "KL/rejected_KL_mean": -112.40410614013672, + "KL/std": 50.249961853027344, + "epoch": 0.6031746031746031, + "fcm_dpo/beta": 0.01826975867152214, + "fcm_dpo/delta": -0.44884711503982544, + "fcm_dpo/margin": 46.246070861816406, + "fcm_dpo/q_t": 0.334566593170166, + "grad_norm": 15.789628982543945, + "learning_rate": 2.052864371672457e-07, + "logits/chosen": 0.5498577356338501, + "logits/rejected": 0.4003596305847168, + "logps/chosen": -134.46482849121094, + "logps/ref_chosen": -68.30679321289062, + "logps/ref_rejected": -113.2708511352539, + "logps/rejected": -225.67495727539062, + "loss": 0.8935, + "margin_dpo/margin_mean": 46.24606704711914, + "margin_dpo/margin_std": 53.64789581298828, + "step": 399 + }, + { + "KL/chosen_KL_mean": -73.99244689941406, + "KL/mean": -87.6726303100586, + "KL/rejected_KL_mean": -101.35281372070312, + "KL/std": 49.09087371826172, + "epoch": 0.6046863189720333, + "fcm_dpo/beta": 0.01818830706179142, + "fcm_dpo/delta": -0.09844368696212769, + "fcm_dpo/margin": 27.36035919189453, + "fcm_dpo/q_t": 0.3985748589038849, + "grad_norm": 25.85154151916504, + "learning_rate": 2.0398616948569493e-07, + "logits/chosen": 0.594917356967926, + "logits/rejected": 0.5359581708908081, + "logps/chosen": -145.6189422607422, + "logps/ref_chosen": -71.62649536132812, + "logps/ref_rejected": -90.98765563964844, + "logps/rejected": -192.34046936035156, + "loss": 1.122, + "margin_dpo/margin_mean": 27.360363006591797, + "margin_dpo/margin_std": 47.804222106933594, + "step": 400 + }, + { + "KL/chosen_KL_mean": -55.45112228393555, + "KL/mean": -76.60586547851562, + "KL/rejected_KL_mean": -97.76061248779297, + "KL/std": 52.43171691894531, + "epoch": 0.6061980347694633, + "fcm_dpo/beta": 0.018093235790729523, + "fcm_dpo/delta": -0.3684466481208801, + "fcm_dpo/margin": 42.30949783325195, + "fcm_dpo/q_t": 0.3453938364982605, + "grad_norm": 12.085314750671387, + "learning_rate": 2.0268718890989752e-07, + "logits/chosen": 0.6513961553573608, + "logits/rejected": 0.548406720161438, + "logps/chosen": -109.17607879638672, + "logps/ref_chosen": -53.72495651245117, + "logps/ref_rejected": -75.06304931640625, + "logps/rejected": -172.82366943359375, + "loss": 0.9183, + "margin_dpo/margin_mean": 42.30949401855469, + "margin_dpo/margin_std": 48.247032165527344, + "step": 401 + }, + { + "KL/chosen_KL_mean": -59.64924621582031, + "KL/mean": -76.43643188476562, + "KL/rejected_KL_mean": -93.22361755371094, + "KL/std": 45.66023254394531, + "epoch": 0.6077097505668935, + "fcm_dpo/beta": 0.017993375658988953, + "fcm_dpo/delta": -0.20568539202213287, + "fcm_dpo/margin": 33.574378967285156, + "fcm_dpo/q_t": 0.3754221200942993, + "grad_norm": 15.690427780151367, + "learning_rate": 2.013895317751323e-07, + "logits/chosen": 0.6434378623962402, + "logits/rejected": 0.6087584495544434, + "logps/chosen": -121.52317810058594, + "logps/ref_chosen": -61.873931884765625, + "logps/ref_rejected": -66.15198516845703, + "logps/rejected": -159.3756103515625, + "loss": 1.0509, + "margin_dpo/margin_mean": 33.574378967285156, + "margin_dpo/margin_std": 49.69902038574219, + "step": 402 + }, + { + "KL/chosen_KL_mean": -63.91669464111328, + "KL/mean": -86.1575927734375, + "KL/rejected_KL_mean": -108.39849090576172, + "KL/std": 52.632625579833984, + "epoch": 0.6092214663643235, + "fcm_dpo/beta": 0.017886489629745483, + "fcm_dpo/delta": -0.39873138070106506, + "fcm_dpo/margin": 44.4818000793457, + "fcm_dpo/q_t": 0.343136191368103, + "grad_norm": 15.241552352905273, + "learning_rate": 2.0009323437965898e-07, + "logits/chosen": 0.7365424633026123, + "logits/rejected": 0.6493263244628906, + "logps/chosen": -115.23818969726562, + "logps/ref_chosen": -51.321502685546875, + "logps/ref_rejected": -86.54010772705078, + "logps/rejected": -194.9385986328125, + "loss": 0.9351, + "margin_dpo/margin_mean": 44.4818000793457, + "margin_dpo/margin_std": 54.205875396728516, + "step": 403 + }, + { + "KL/chosen_KL_mean": -60.81599426269531, + "KL/mean": -81.69652557373047, + "KL/rejected_KL_mean": -102.57705688476562, + "KL/std": 50.732635498046875, + "epoch": 0.6107331821617535, + "fcm_dpo/beta": 0.017737917602062225, + "fcm_dpo/delta": -0.3438475430011749, + "fcm_dpo/margin": 41.761070251464844, + "fcm_dpo/q_t": 0.3538442850112915, + "grad_norm": 18.1308650970459, + "learning_rate": 1.9879833298370237e-07, + "logits/chosen": 0.6057857275009155, + "logits/rejected": 0.5072075724601746, + "logps/chosen": -123.0788803100586, + "logps/ref_chosen": -62.26288604736328, + "logps/ref_rejected": -95.19029998779297, + "logps/rejected": -197.76736450195312, + "loss": 0.9818, + "margin_dpo/margin_mean": 41.761070251464844, + "margin_dpo/margin_std": 54.85755157470703, + "step": 404 + }, + { + "KL/chosen_KL_mean": -63.13936233520508, + "KL/mean": -79.79869079589844, + "KL/rejected_KL_mean": -96.45801544189453, + "KL/std": 48.215606689453125, + "epoch": 0.6122448979591837, + "fcm_dpo/beta": 0.01765264943242073, + "fcm_dpo/delta": -0.1898302137851715, + "fcm_dpo/margin": 33.31864929199219, + "fcm_dpo/q_t": 0.38051706552505493, + "grad_norm": 14.231498718261719, + "learning_rate": 1.975048638084379e-07, + "logits/chosen": 0.6734004616737366, + "logits/rejected": 0.6236171722412109, + "logps/chosen": -113.72370147705078, + "logps/ref_chosen": -50.5843391418457, + "logps/ref_rejected": -65.43156433105469, + "logps/rejected": -161.88958740234375, + "loss": 1.0422, + "margin_dpo/margin_mean": 33.31864547729492, + "margin_dpo/margin_std": 46.35669708251953, + "step": 405 + }, + { + "KL/chosen_KL_mean": -61.924461364746094, + "KL/mean": -82.48919677734375, + "KL/rejected_KL_mean": -103.05392456054688, + "KL/std": 50.14469528198242, + "epoch": 0.6137566137566137, + "fcm_dpo/beta": 0.017570994794368744, + "fcm_dpo/delta": -0.3250589370727539, + "fcm_dpo/margin": 41.129478454589844, + "fcm_dpo/q_t": 0.3523600995540619, + "grad_norm": 15.564558029174805, + "learning_rate": 1.9621286303497914e-07, + "logits/chosen": 0.652951180934906, + "logits/rejected": 0.4872160851955414, + "logps/chosen": -110.92005920410156, + "logps/ref_chosen": -48.99560546875, + "logps/ref_rejected": -92.47774505615234, + "logps/rejected": -195.53167724609375, + "loss": 0.9868, + "margin_dpo/margin_mean": 41.129478454589844, + "margin_dpo/margin_std": 54.664302825927734, + "step": 406 + }, + { + "KL/chosen_KL_mean": -76.57392883300781, + "KL/mean": -93.85360717773438, + "KL/rejected_KL_mean": -111.13328552246094, + "KL/std": 56.542171478271484, + "epoch": 0.6152683295540439, + "fcm_dpo/beta": 0.017491009086370468, + "fcm_dpo/delta": -0.2057061642408371, + "fcm_dpo/margin": 34.559349060058594, + "fcm_dpo/q_t": 0.3769576847553253, + "grad_norm": 16.18971824645996, + "learning_rate": 1.9492236680336483e-07, + "logits/chosen": 0.5140019655227661, + "logits/rejected": 0.4366782307624817, + "logps/chosen": -165.9744873046875, + "logps/ref_chosen": -89.40056610107422, + "logps/ref_rejected": -99.28775024414062, + "logps/rejected": -210.42105102539062, + "loss": 1.0556, + "margin_dpo/margin_mean": 34.559349060058594, + "margin_dpo/margin_std": 53.048912048339844, + "step": 407 + }, + { + "KL/chosen_KL_mean": -56.734642028808594, + "KL/mean": -80.69111633300781, + "KL/rejected_KL_mean": -104.64759826660156, + "KL/std": 48.3653564453125, + "epoch": 0.6167800453514739, + "fcm_dpo/beta": 0.01737467385828495, + "fcm_dpo/delta": -0.43586891889572144, + "fcm_dpo/margin": 47.91295623779297, + "fcm_dpo/q_t": 0.329407274723053, + "grad_norm": 13.42773151397705, + "learning_rate": 1.9363341121154895e-07, + "logits/chosen": 0.6290972232818604, + "logits/rejected": 0.5435121059417725, + "logps/chosen": -111.43855285644531, + "logps/ref_chosen": -54.70391845703125, + "logps/ref_rejected": -73.98648834228516, + "logps/rejected": -178.63409423828125, + "loss": 0.8802, + "margin_dpo/margin_mean": 47.91295623779297, + "margin_dpo/margin_std": 49.45269775390625, + "step": 408 + }, + { + "KL/chosen_KL_mean": -69.91849517822266, + "KL/mean": -82.78309631347656, + "KL/rejected_KL_mean": -95.64772033691406, + "KL/std": 44.54106140136719, + "epoch": 0.618291761148904, + "fcm_dpo/beta": 0.017304107546806335, + "fcm_dpo/delta": -0.04544477164745331, + "fcm_dpo/margin": 25.729223251342773, + "fcm_dpo/q_t": 0.41034555435180664, + "grad_norm": 17.73507308959961, + "learning_rate": 1.9234603231438994e-07, + "logits/chosen": 0.6348155736923218, + "logits/rejected": 0.6412778496742249, + "logps/chosen": -132.03671264648438, + "logps/ref_chosen": -62.11822509765625, + "logps/ref_rejected": -61.933509826660156, + "logps/rejected": -157.58123779296875, + "loss": 1.16, + "margin_dpo/margin_mean": 25.729225158691406, + "margin_dpo/margin_std": 50.80590057373047, + "step": 409 + }, + { + "KL/chosen_KL_mean": -65.20570373535156, + "KL/mean": -84.81724548339844, + "KL/rejected_KL_mean": -104.42878723144531, + "KL/std": 47.78340148925781, + "epoch": 0.6198034769463341, + "fcm_dpo/beta": 0.017215661704540253, + "fcm_dpo/delta": -0.2775804102420807, + "fcm_dpo/margin": 39.22309112548828, + "fcm_dpo/q_t": 0.3554009795188904, + "grad_norm": 15.138179779052734, + "learning_rate": 1.9106026612264315e-07, + "logits/chosen": 0.6383532285690308, + "logits/rejected": 0.6108101606369019, + "logps/chosen": -127.00836181640625, + "logps/ref_chosen": -61.80266189575195, + "logps/ref_rejected": -76.60002136230469, + "logps/rejected": -181.02880859375, + "loss": 0.9384, + "margin_dpo/margin_mean": 39.22309112548828, + "margin_dpo/margin_std": 41.84228515625, + "step": 410 + }, + { + "KL/chosen_KL_mean": -73.12362670898438, + "KL/mean": -93.15718078613281, + "KL/rejected_KL_mean": -113.19075012207031, + "KL/std": 54.41527557373047, + "epoch": 0.6213151927437641, + "fcm_dpo/beta": 0.01713244616985321, + "fcm_dpo/delta": -0.2884591817855835, + "fcm_dpo/margin": 40.067138671875, + "fcm_dpo/q_t": 0.3615899682044983, + "grad_norm": 14.97659683227539, + "learning_rate": 1.8977614860195296e-07, + "logits/chosen": 0.6429640650749207, + "logits/rejected": 0.5793659090995789, + "logps/chosen": -127.56901550292969, + "logps/ref_chosen": -54.44539260864258, + "logps/ref_rejected": -74.5650863647461, + "logps/rejected": -187.75582885742188, + "loss": 1.0002, + "margin_dpo/margin_mean": 40.067138671875, + "margin_dpo/margin_std": 55.26818084716797, + "step": 411 + }, + { + "KL/chosen_KL_mean": -75.49935913085938, + "KL/mean": -93.84625244140625, + "KL/rejected_KL_mean": -112.19314575195312, + "KL/std": 46.57638931274414, + "epoch": 0.6228269085411943, + "fcm_dpo/beta": 0.01704230345785618, + "fcm_dpo/delta": -0.2269406020641327, + "fcm_dpo/margin": 36.69379425048828, + "fcm_dpo/q_t": 0.3701617121696472, + "grad_norm": 15.08652400970459, + "learning_rate": 1.8849371567184662e-07, + "logits/chosen": 0.6498918533325195, + "logits/rejected": 0.5774151086807251, + "logps/chosen": -130.7474365234375, + "logps/ref_chosen": -55.248085021972656, + "logps/ref_rejected": -68.96623229980469, + "logps/rejected": -181.1593780517578, + "loss": 0.9971, + "margin_dpo/margin_mean": 36.693790435791016, + "margin_dpo/margin_std": 47.453514099121094, + "step": 412 + }, + { + "KL/chosen_KL_mean": -82.62654113769531, + "KL/mean": -99.52752685546875, + "KL/rejected_KL_mean": -116.42852783203125, + "KL/std": 49.28752899169922, + "epoch": 0.6243386243386243, + "fcm_dpo/beta": 0.01698903553187847, + "fcm_dpo/delta": -0.17525365948677063, + "fcm_dpo/margin": 33.802001953125, + "fcm_dpo/q_t": 0.3831592798233032, + "grad_norm": 17.580890655517578, + "learning_rate": 1.872130032047302e-07, + "logits/chosen": 0.4469856023788452, + "logits/rejected": 0.4094501733779907, + "logps/chosen": -151.3472900390625, + "logps/ref_chosen": -68.72074890136719, + "logps/ref_rejected": -78.76539611816406, + "logps/rejected": -195.1939239501953, + "loss": 1.1124, + "margin_dpo/margin_mean": 33.801998138427734, + "margin_dpo/margin_std": 59.33550262451172, + "step": 413 + }, + { + "KL/chosen_KL_mean": -75.0484390258789, + "KL/mean": -95.25251770019531, + "KL/rejected_KL_mean": -115.45659637451172, + "KL/std": 56.00800323486328, + "epoch": 0.6258503401360545, + "fcm_dpo/beta": 0.016899818554520607, + "fcm_dpo/delta": -0.2848397493362427, + "fcm_dpo/margin": 40.408164978027344, + "fcm_dpo/q_t": 0.35738152265548706, + "grad_norm": 15.938345909118652, + "learning_rate": 1.8593404702488436e-07, + "logits/chosen": 0.6080368757247925, + "logits/rejected": 0.5434196591377258, + "logps/chosen": -129.1866455078125, + "logps/ref_chosen": -54.138214111328125, + "logps/ref_rejected": -74.65741729736328, + "logps/rejected": -190.114013671875, + "loss": 0.9753, + "margin_dpo/margin_mean": 40.408164978027344, + "margin_dpo/margin_std": 50.767784118652344, + "step": 414 + }, + { + "KL/chosen_KL_mean": -72.423095703125, + "KL/mean": -90.89973449707031, + "KL/rejected_KL_mean": -109.37638092041016, + "KL/std": 54.18439865112305, + "epoch": 0.6273620559334845, + "fcm_dpo/beta": 0.016815291717648506, + "fcm_dpo/delta": -0.2228357195854187, + "fcm_dpo/margin": 36.953285217285156, + "fcm_dpo/q_t": 0.37174832820892334, + "grad_norm": 15.645916938781738, + "learning_rate": 1.846568829074628e-07, + "logits/chosen": 0.6561309099197388, + "logits/rejected": 0.6361061334609985, + "logps/chosen": -128.34164428710938, + "logps/ref_chosen": -55.91856002807617, + "logps/ref_rejected": -61.747703552246094, + "logps/rejected": -171.12408447265625, + "loss": 1.0438, + "margin_dpo/margin_mean": 36.953285217285156, + "margin_dpo/margin_std": 55.12226867675781, + "step": 415 + }, + { + "KL/chosen_KL_mean": -78.98628234863281, + "KL/mean": -93.94440460205078, + "KL/rejected_KL_mean": -108.90253448486328, + "KL/std": 55.698997497558594, + "epoch": 0.6288737717309146, + "fcm_dpo/beta": 0.016758650541305542, + "fcm_dpo/delta": -0.25184109807014465, + "fcm_dpo/margin": 29.916240692138672, + "fcm_dpo/q_t": 0.40063685178756714, + "grad_norm": 17.139232635498047, + "learning_rate": 1.8338154657749128e-07, + "logits/chosen": 0.6013250350952148, + "logits/rejected": 0.5480623841285706, + "logps/chosen": -133.70936584472656, + "logps/ref_chosen": -54.72308349609375, + "logps/ref_rejected": -69.17388916015625, + "logps/rejected": -178.076416015625, + "loss": 1.1481, + "margin_dpo/margin_mean": 29.916240692138672, + "margin_dpo/margin_std": 52.974891662597656, + "step": 416 + }, + { + "KL/chosen_KL_mean": -82.40274810791016, + "KL/mean": -102.98678588867188, + "KL/rejected_KL_mean": -123.57081604003906, + "KL/std": 51.19279479980469, + "epoch": 0.6303854875283447, + "fcm_dpo/beta": 0.016657628118991852, + "fcm_dpo/delta": -0.2876332402229309, + "fcm_dpo/margin": 41.168060302734375, + "fcm_dpo/q_t": 0.35524600744247437, + "grad_norm": 15.734081268310547, + "learning_rate": 1.8210807370886849e-07, + "logits/chosen": 0.7438496351242065, + "logits/rejected": 0.6693851947784424, + "logps/chosen": -139.1940155029297, + "logps/ref_chosen": -56.791259765625, + "logps/ref_rejected": -68.7791748046875, + "logps/rejected": -192.34999084472656, + "loss": 1.0106, + "margin_dpo/margin_mean": 41.168060302734375, + "margin_dpo/margin_std": 56.998626708984375, + "step": 417 + }, + { + "KL/chosen_KL_mean": -89.2573471069336, + "KL/mean": -104.68053436279297, + "KL/rejected_KL_mean": -120.10371398925781, + "KL/std": 54.627742767333984, + "epoch": 0.6318972033257747, + "fcm_dpo/beta": 0.01655709557235241, + "fcm_dpo/delta": -0.19127853214740753, + "fcm_dpo/margin": 30.846370697021484, + "fcm_dpo/q_t": 0.39703673124313354, + "grad_norm": 18.322853088378906, + "learning_rate": 1.8083649992336825e-07, + "logits/chosen": 0.654333770275116, + "logits/rejected": 0.6596550941467285, + "logps/chosen": -158.36532592773438, + "logps/ref_chosen": -69.10798645019531, + "logps/ref_rejected": -75.09132385253906, + "logps/rejected": -195.19503784179688, + "loss": 1.1308, + "margin_dpo/margin_mean": 30.846370697021484, + "margin_dpo/margin_std": 55.0068359375, + "step": 418 + }, + { + "KL/chosen_KL_mean": -69.68336486816406, + "KL/mean": -92.48382568359375, + "KL/rejected_KL_mean": -115.2842788696289, + "KL/std": 53.37661361694336, + "epoch": 0.6334089191232048, + "fcm_dpo/beta": 0.016465239226818085, + "fcm_dpo/delta": -0.35359275341033936, + "fcm_dpo/margin": 45.60090637207031, + "fcm_dpo/q_t": 0.3505327105522156, + "grad_norm": 15.882049560546875, + "learning_rate": 1.7956686078964255e-07, + "logits/chosen": 0.5413916110992432, + "logits/rejected": 0.4839738607406616, + "logps/chosen": -127.85513305664062, + "logps/ref_chosen": -58.1717643737793, + "logps/ref_rejected": -71.67066955566406, + "logps/rejected": -186.9549560546875, + "loss": 0.9559, + "margin_dpo/margin_mean": 45.60090637207031, + "margin_dpo/margin_std": 57.791778564453125, + "step": 419 + }, + { + "KL/chosen_KL_mean": -90.16976928710938, + "KL/mean": -102.14017486572266, + "KL/rejected_KL_mean": -114.11058044433594, + "KL/std": 54.92378616333008, + "epoch": 0.6349206349206349, + "fcm_dpo/beta": 0.016431353986263275, + "fcm_dpo/delta": 0.00656590610742569, + "fcm_dpo/margin": 23.940799713134766, + "fcm_dpo/q_t": 0.4205707609653473, + "grad_norm": 16.866971969604492, + "learning_rate": 1.782991918222275e-07, + "logits/chosen": 0.6038463115692139, + "logits/rejected": 0.5543715953826904, + "logps/chosen": -147.22328186035156, + "logps/ref_chosen": -57.05351257324219, + "logps/ref_rejected": -62.670982360839844, + "logps/rejected": -176.78155517578125, + "loss": 1.2534, + "margin_dpo/margin_mean": 23.940799713134766, + "margin_dpo/margin_std": 60.628318786621094, + "step": 420 + }, + { + "KL/chosen_KL_mean": -84.62358856201172, + "KL/mean": -101.34190368652344, + "KL/rejected_KL_mean": -118.06021881103516, + "KL/std": 54.297325134277344, + "epoch": 0.636432350718065, + "fcm_dpo/beta": 0.016398075968027115, + "fcm_dpo/delta": -0.1492297202348709, + "fcm_dpo/margin": 33.43663024902344, + "fcm_dpo/q_t": 0.39096736907958984, + "grad_norm": 18.37002182006836, + "learning_rate": 1.7703352848054887e-07, + "logits/chosen": 0.6103418469429016, + "logits/rejected": 0.5464081764221191, + "logps/chosen": -141.94683837890625, + "logps/ref_chosen": -57.32324981689453, + "logps/ref_rejected": -75.33782958984375, + "logps/rejected": -193.39804077148438, + "loss": 1.1679, + "margin_dpo/margin_mean": 33.43663024902344, + "margin_dpo/margin_std": 66.37718200683594, + "step": 421 + }, + { + "KL/chosen_KL_mean": -72.61814880371094, + "KL/mean": -94.66935729980469, + "KL/rejected_KL_mean": -116.72056579589844, + "KL/std": 51.81633377075195, + "epoch": 0.6379440665154951, + "fcm_dpo/beta": 0.01632114127278328, + "fcm_dpo/delta": -0.32202476263046265, + "fcm_dpo/margin": 44.10242462158203, + "fcm_dpo/q_t": 0.3516920506954193, + "grad_norm": 17.09633445739746, + "learning_rate": 1.7576990616793137e-07, + "logits/chosen": 0.614160418510437, + "logits/rejected": 0.5957762598991394, + "logps/chosen": -139.67572021484375, + "logps/ref_chosen": -67.05757141113281, + "logps/ref_rejected": -72.12803649902344, + "logps/rejected": -188.84860229492188, + "loss": 0.9549, + "margin_dpo/margin_mean": 44.102420806884766, + "margin_dpo/margin_std": 53.641441345214844, + "step": 422 + }, + { + "KL/chosen_KL_mean": -76.13412475585938, + "KL/mean": -98.8819580078125, + "KL/rejected_KL_mean": -121.6297836303711, + "KL/std": 56.02351379394531, + "epoch": 0.6394557823129252, + "fcm_dpo/beta": 0.016200382262468338, + "fcm_dpo/delta": -0.3396818935871124, + "fcm_dpo/margin": 45.495662689208984, + "fcm_dpo/q_t": 0.3516571521759033, + "grad_norm": 14.82790470123291, + "learning_rate": 1.745083602306071e-07, + "logits/chosen": 0.6792205572128296, + "logits/rejected": 0.6034238338470459, + "logps/chosen": -130.19580078125, + "logps/ref_chosen": -54.06167221069336, + "logps/ref_rejected": -76.64092254638672, + "logps/rejected": -198.2707061767578, + "loss": 0.9513, + "margin_dpo/margin_mean": 45.49565887451172, + "margin_dpo/margin_std": 56.05955505371094, + "step": 423 + }, + { + "KL/chosen_KL_mean": -78.58985900878906, + "KL/mean": -102.63473510742188, + "KL/rejected_KL_mean": -126.67961120605469, + "KL/std": 53.19892120361328, + "epoch": 0.6409674981103552, + "fcm_dpo/beta": 0.01607362926006317, + "fcm_dpo/delta": -0.3763389587402344, + "fcm_dpo/margin": 48.08976745605469, + "fcm_dpo/q_t": 0.343904972076416, + "grad_norm": 17.946325302124023, + "learning_rate": 1.7324892595672804e-07, + "logits/chosen": 0.5570648312568665, + "logits/rejected": 0.5156468152999878, + "logps/chosen": -132.19873046875, + "logps/ref_chosen": -53.60887145996094, + "logps/ref_rejected": -79.2139892578125, + "logps/rejected": -205.89361572265625, + "loss": 0.9306, + "margin_dpo/margin_mean": 48.08976745605469, + "margin_dpo/margin_std": 56.2728271484375, + "step": 424 + }, + { + "KL/chosen_KL_mean": -78.46670532226562, + "KL/mean": -96.44735717773438, + "KL/rejected_KL_mean": -114.42801666259766, + "KL/std": 53.36773681640625, + "epoch": 0.6424792139077853, + "fcm_dpo/beta": 0.01601138338446617, + "fcm_dpo/delta": -0.1768464893102646, + "fcm_dpo/margin": 35.96131896972656, + "fcm_dpo/q_t": 0.3824845552444458, + "grad_norm": 16.53751564025879, + "learning_rate": 1.7199163857537824e-07, + "logits/chosen": 0.7052649259567261, + "logits/rejected": 0.6726754903793335, + "logps/chosen": -136.88137817382812, + "logps/ref_chosen": -58.41468048095703, + "logps/ref_rejected": -66.59054565429688, + "logps/rejected": -181.0185546875, + "loss": 1.0648, + "margin_dpo/margin_mean": 35.96131896972656, + "margin_dpo/margin_std": 55.85108947753906, + "step": 425 + }, + { + "KL/chosen_KL_mean": -95.75052642822266, + "KL/mean": -107.16114807128906, + "KL/rejected_KL_mean": -118.57176208496094, + "KL/std": 53.0162353515625, + "epoch": 0.6439909297052154, + "fcm_dpo/beta": 0.01600591093301773, + "fcm_dpo/delta": 0.034853242337703705, + "fcm_dpo/margin": 22.82123374938965, + "fcm_dpo/q_t": 0.42325669527053833, + "grad_norm": 21.665000915527344, + "learning_rate": 1.7073653325558828e-07, + "logits/chosen": 0.5796278715133667, + "logits/rejected": 0.5866859555244446, + "logps/chosen": -167.458740234375, + "logps/ref_chosen": -71.70822143554688, + "logps/ref_rejected": -73.57725524902344, + "logps/rejected": -192.14901733398438, + "loss": 1.2766, + "margin_dpo/margin_mean": 22.82123374938965, + "margin_dpo/margin_std": 63.02619171142578, + "step": 426 + }, + { + "KL/chosen_KL_mean": -91.18772888183594, + "KL/mean": -109.51598358154297, + "KL/rejected_KL_mean": -127.84423828125, + "KL/std": 58.52879333496094, + "epoch": 0.6455026455026455, + "fcm_dpo/beta": 0.015966089442372322, + "fcm_dpo/delta": -0.186357319355011, + "fcm_dpo/margin": 36.6565055847168, + "fcm_dpo/q_t": 0.38394221663475037, + "grad_norm": 17.36440086364746, + "learning_rate": 1.6948364510535218e-07, + "logits/chosen": 0.6597490310668945, + "logits/rejected": 0.5971434116363525, + "logps/chosen": -149.83050537109375, + "logps/ref_chosen": -58.64276885986328, + "logps/ref_rejected": -86.25437927246094, + "logps/rejected": -214.09861755371094, + "loss": 1.1, + "margin_dpo/margin_mean": 36.6565055847168, + "margin_dpo/margin_std": 63.468605041503906, + "step": 427 + }, + { + "KL/chosen_KL_mean": -84.71063232421875, + "KL/mean": -106.85235595703125, + "KL/rejected_KL_mean": -128.99407958984375, + "KL/std": 59.434532165527344, + "epoch": 0.6470143613000756, + "fcm_dpo/beta": 0.015889186412096024, + "fcm_dpo/delta": -0.30566510558128357, + "fcm_dpo/margin": 44.28342819213867, + "fcm_dpo/q_t": 0.36132287979125977, + "grad_norm": 15.740050315856934, + "learning_rate": 1.6823300917064458e-07, + "logits/chosen": 0.6053894758224487, + "logits/rejected": 0.5555422306060791, + "logps/chosen": -151.30667114257812, + "logps/ref_chosen": -66.5960464477539, + "logps/ref_rejected": -82.3941650390625, + "logps/rejected": -211.38824462890625, + "loss": 1.0083, + "margin_dpo/margin_mean": 44.28343200683594, + "margin_dpo/margin_std": 62.53662109375, + "step": 428 + }, + { + "KL/chosen_KL_mean": -89.94075012207031, + "KL/mean": -107.12989807128906, + "KL/rejected_KL_mean": -124.31904602050781, + "KL/std": 53.27215576171875, + "epoch": 0.6485260770975056, + "fcm_dpo/beta": 0.015830520540475845, + "fcm_dpo/delta": -0.14500769972801208, + "fcm_dpo/margin": 34.3782958984375, + "fcm_dpo/q_t": 0.3869292140007019, + "grad_norm": 17.362619400024414, + "learning_rate": 1.669846604344412e-07, + "logits/chosen": 0.5725862979888916, + "logits/rejected": 0.5897176265716553, + "logps/chosen": -146.95045471191406, + "logps/ref_chosen": -57.00970458984375, + "logps/ref_rejected": -59.86549377441406, + "logps/rejected": -184.18453979492188, + "loss": 1.1033, + "margin_dpo/margin_mean": 34.3782958984375, + "margin_dpo/margin_std": 57.47087097167969, + "step": 429 + }, + { + "KL/chosen_KL_mean": -81.42185974121094, + "KL/mean": -105.9416275024414, + "KL/rejected_KL_mean": -130.46139526367188, + "KL/std": 54.26333236694336, + "epoch": 0.6500377928949358, + "fcm_dpo/beta": 0.01571904495358467, + "fcm_dpo/delta": -0.37376558780670166, + "fcm_dpo/margin": 49.039527893066406, + "fcm_dpo/q_t": 0.34044498205184937, + "grad_norm": 14.907088279724121, + "learning_rate": 1.6573863381573954e-07, + "logits/chosen": 0.5389697551727295, + "logits/rejected": 0.5364863872528076, + "logps/chosen": -140.9850616455078, + "logps/ref_chosen": -59.563194274902344, + "logps/ref_rejected": -70.52289581298828, + "logps/rejected": -200.98428344726562, + "loss": 0.9323, + "margin_dpo/margin_mean": 49.039527893066406, + "margin_dpo/margin_std": 57.40979766845703, + "step": 430 + }, + { + "KL/chosen_KL_mean": -75.9009017944336, + "KL/mean": -94.57374572753906, + "KL/rejected_KL_mean": -113.24659729003906, + "KL/std": 55.441612243652344, + "epoch": 0.6515495086923658, + "fcm_dpo/beta": 0.015644002705812454, + "fcm_dpo/delta": -0.18536588549613953, + "fcm_dpo/margin": 37.3456916809082, + "fcm_dpo/q_t": 0.3779263496398926, + "grad_norm": 14.4020414352417, + "learning_rate": 1.6449496416858282e-07, + "logits/chosen": 0.5917558670043945, + "logits/rejected": 0.5395553112030029, + "logps/chosen": -126.10122680664062, + "logps/ref_chosen": -50.20032501220703, + "logps/ref_rejected": -77.81680297851562, + "logps/rejected": -191.0634002685547, + "loss": 1.05, + "margin_dpo/margin_mean": 37.34568786621094, + "margin_dpo/margin_std": 55.41195297241211, + "step": 431 + }, + { + "KL/chosen_KL_mean": -78.02108764648438, + "KL/mean": -97.24517822265625, + "KL/rejected_KL_mean": -116.46928405761719, + "KL/std": 51.37657165527344, + "epoch": 0.6530612244897959, + "fcm_dpo/beta": 0.015583939850330353, + "fcm_dpo/delta": -0.20040223002433777, + "fcm_dpo/margin": 38.44818878173828, + "fcm_dpo/q_t": 0.37600746750831604, + "grad_norm": 15.461301803588867, + "learning_rate": 1.632536862810844e-07, + "logits/chosen": 0.6558856964111328, + "logits/rejected": 0.6033505797386169, + "logps/chosen": -139.683837890625, + "logps/ref_chosen": -61.662757873535156, + "logps/ref_rejected": -83.94496154785156, + "logps/rejected": -200.41424560546875, + "loss": 1.0581, + "margin_dpo/margin_mean": 38.44818878173828, + "margin_dpo/margin_std": 59.100425720214844, + "step": 432 + }, + { + "KL/chosen_KL_mean": -78.08153533935547, + "KL/mean": -102.20304870605469, + "KL/rejected_KL_mean": -126.3245849609375, + "KL/std": 54.143951416015625, + "epoch": 0.654572940287226, + "fcm_dpo/beta": 0.015497459098696709, + "fcm_dpo/delta": -0.3501299321651459, + "fcm_dpo/margin": 48.2430419921875, + "fcm_dpo/q_t": 0.3454676866531372, + "grad_norm": 15.343486785888672, + "learning_rate": 1.6201483487445515e-07, + "logits/chosen": 0.7134816646575928, + "logits/rejected": 0.7090972661972046, + "logps/chosen": -141.8107147216797, + "logps/ref_chosen": -63.72917938232422, + "logps/ref_rejected": -65.8391342163086, + "logps/rejected": -192.16371154785156, + "loss": 0.9512, + "margin_dpo/margin_mean": 48.2430419921875, + "margin_dpo/margin_std": 57.79951477050781, + "step": 433 + }, + { + "KL/chosen_KL_mean": -69.61598205566406, + "KL/mean": -92.99039459228516, + "KL/rejected_KL_mean": -116.36482238769531, + "KL/std": 53.780303955078125, + "epoch": 0.656084656084656, + "fcm_dpo/beta": 0.015374215319752693, + "fcm_dpo/delta": -0.321450412273407, + "fcm_dpo/margin": 46.74883270263672, + "fcm_dpo/q_t": 0.35029372572898865, + "grad_norm": 14.19492244720459, + "learning_rate": 1.6077844460203204e-07, + "logits/chosen": 0.7226836681365967, + "logits/rejected": 0.6564086675643921, + "logps/chosen": -117.58929443359375, + "logps/ref_chosen": -47.97331619262695, + "logps/ref_rejected": -72.51132202148438, + "logps/rejected": -188.8761444091797, + "loss": 1.0061, + "margin_dpo/margin_mean": 46.74884033203125, + "margin_dpo/margin_std": 63.83148193359375, + "step": 434 + }, + { + "KL/chosen_KL_mean": -81.0145263671875, + "KL/mean": -101.52911376953125, + "KL/rejected_KL_mean": -122.043701171875, + "KL/std": 54.32355880737305, + "epoch": 0.6575963718820862, + "fcm_dpo/beta": 0.015319978818297386, + "fcm_dpo/delta": -0.2299482673406601, + "fcm_dpo/margin": 41.02918243408203, + "fcm_dpo/q_t": 0.37109190225601196, + "grad_norm": 17.728330612182617, + "learning_rate": 1.5954455004830878e-07, + "logits/chosen": 0.7391527891159058, + "logits/rejected": 0.6977601051330566, + "logps/chosen": -138.07476806640625, + "logps/ref_chosen": -57.06024932861328, + "logps/ref_rejected": -71.69146728515625, + "logps/rejected": -193.73516845703125, + "loss": 1.0349, + "margin_dpo/margin_mean": 41.02918243408203, + "margin_dpo/margin_std": 58.975990295410156, + "step": 435 + }, + { + "KL/chosen_KL_mean": -78.7829818725586, + "KL/mean": -97.46292114257812, + "KL/rejected_KL_mean": -116.14283752441406, + "KL/std": 54.9493408203125, + "epoch": 0.6591080876795162, + "fcm_dpo/beta": 0.015253517776727676, + "fcm_dpo/delta": -0.17083582282066345, + "fcm_dpo/margin": 37.35985565185547, + "fcm_dpo/q_t": 0.3830685019493103, + "grad_norm": 15.47319507598877, + "learning_rate": 1.5831318572796847e-07, + "logits/chosen": 0.6517062187194824, + "logits/rejected": 0.5936406850814819, + "logps/chosen": -134.9410400390625, + "logps/ref_chosen": -56.158050537109375, + "logps/ref_rejected": -67.63787841796875, + "logps/rejected": -183.7807159423828, + "loss": 1.0866, + "margin_dpo/margin_mean": 37.35985565185547, + "margin_dpo/margin_std": 61.21410369873047, + "step": 436 + }, + { + "KL/chosen_KL_mean": -85.97834777832031, + "KL/mean": -104.84724426269531, + "KL/rejected_KL_mean": -123.71614074707031, + "KL/std": 55.415348052978516, + "epoch": 0.6606198034769464, + "fcm_dpo/beta": 0.015178972855210304, + "fcm_dpo/delta": -0.17444270849227905, + "fcm_dpo/margin": 37.737789154052734, + "fcm_dpo/q_t": 0.38490501046180725, + "grad_norm": 17.665002822875977, + "learning_rate": 1.5708438608491815e-07, + "logits/chosen": 0.6378095746040344, + "logits/rejected": 0.5026010274887085, + "logps/chosen": -142.96414184570312, + "logps/ref_chosen": -56.98578643798828, + "logps/ref_rejected": -85.61524963378906, + "logps/rejected": -209.33139038085938, + "loss": 1.1252, + "margin_dpo/margin_mean": 37.737789154052734, + "margin_dpo/margin_std": 66.06404113769531, + "step": 437 + }, + { + "KL/chosen_KL_mean": -74.57269287109375, + "KL/mean": -100.32266998291016, + "KL/rejected_KL_mean": -126.0726547241211, + "KL/std": 60.190162658691406, + "epoch": 0.6621315192743764, + "fcm_dpo/beta": 0.015106827020645142, + "fcm_dpo/delta": -0.3808387815952301, + "fcm_dpo/margin": 51.499969482421875, + "fcm_dpo/q_t": 0.34623122215270996, + "grad_norm": 15.793719291687012, + "learning_rate": 1.558581854913253e-07, + "logits/chosen": 0.7130237817764282, + "logits/rejected": 0.6410787105560303, + "logps/chosen": -115.8504638671875, + "logps/ref_chosen": -41.27777862548828, + "logps/ref_rejected": -65.33840942382812, + "logps/rejected": -191.41107177734375, + "loss": 0.9406, + "margin_dpo/margin_mean": 51.499969482421875, + "margin_dpo/margin_std": 63.32539749145508, + "step": 438 + }, + { + "KL/chosen_KL_mean": -81.83445739746094, + "KL/mean": -103.6722412109375, + "KL/rejected_KL_mean": -125.510009765625, + "KL/std": 59.10997009277344, + "epoch": 0.6636432350718064, + "fcm_dpo/beta": 0.01502758078277111, + "fcm_dpo/delta": -0.25796717405319214, + "fcm_dpo/margin": 43.675567626953125, + "fcm_dpo/q_t": 0.36370548605918884, + "grad_norm": 15.497008323669434, + "learning_rate": 1.5463461824665658e-07, + "logits/chosen": 0.5407925844192505, + "logits/rejected": 0.50162672996521, + "logps/chosen": -163.25210571289062, + "logps/ref_chosen": -81.41764831542969, + "logps/ref_rejected": -94.72309875488281, + "logps/rejected": -220.2331085205078, + "loss": 0.9884, + "margin_dpo/margin_mean": 43.675567626953125, + "margin_dpo/margin_std": 55.45774459838867, + "step": 439 + }, + { + "KL/chosen_KL_mean": -74.45477294921875, + "KL/mean": -96.9222412109375, + "KL/rejected_KL_mean": -119.38970947265625, + "KL/std": 53.95646667480469, + "epoch": 0.6651549508692366, + "fcm_dpo/beta": 0.014940877445042133, + "fcm_dpo/delta": -0.2731121778488159, + "fcm_dpo/margin": 44.934932708740234, + "fcm_dpo/q_t": 0.3636067509651184, + "grad_norm": 23.114179611206055, + "learning_rate": 1.534137185767178e-07, + "logits/chosen": 0.6434664130210876, + "logits/rejected": 0.5399199724197388, + "logps/chosen": -116.99295806884766, + "logps/ref_chosen": -42.538185119628906, + "logps/ref_rejected": -69.78813934326172, + "logps/rejected": -189.1778564453125, + "loss": 1.0128, + "margin_dpo/margin_mean": 44.934932708740234, + "margin_dpo/margin_std": 62.5695686340332, + "step": 440 + }, + { + "KL/chosen_KL_mean": -69.44387817382812, + "KL/mean": -94.70149230957031, + "KL/rejected_KL_mean": -119.95911407470703, + "KL/std": 56.708343505859375, + "epoch": 0.6666666666666666, + "fcm_dpo/beta": 0.014826279133558273, + "fcm_dpo/delta": -0.3518187403678894, + "fcm_dpo/margin": 50.515235900878906, + "fcm_dpo/q_t": 0.3417903482913971, + "grad_norm": 15.354460716247559, + "learning_rate": 1.521955206326976e-07, + "logits/chosen": 0.617450475692749, + "logits/rejected": 0.5188318490982056, + "logps/chosen": -127.037109375, + "logps/ref_chosen": -57.593223571777344, + "logps/ref_rejected": -84.82878875732422, + "logps/rejected": -204.78790283203125, + "loss": 0.9005, + "margin_dpo/margin_mean": 50.515235900878906, + "margin_dpo/margin_std": 51.828575134277344, + "step": 441 + }, + { + "KL/chosen_KL_mean": -88.58029174804688, + "KL/mean": -112.20353698730469, + "KL/rejected_KL_mean": -135.8267822265625, + "KL/std": 56.35088348388672, + "epoch": 0.6681783824640968, + "fcm_dpo/beta": 0.014739081263542175, + "fcm_dpo/delta": -0.2985053062438965, + "fcm_dpo/margin": 47.24647521972656, + "fcm_dpo/q_t": 0.35557490587234497, + "grad_norm": 16.09771156311035, + "learning_rate": 1.5098005849021078e-07, + "logits/chosen": 0.6071983575820923, + "logits/rejected": 0.5529823899269104, + "logps/chosen": -156.04150390625, + "logps/ref_chosen": -67.46121978759766, + "logps/ref_rejected": -89.0693588256836, + "logps/rejected": -224.89613342285156, + "loss": 0.9724, + "margin_dpo/margin_mean": 47.24647521972656, + "margin_dpo/margin_std": 59.35338592529297, + "step": 442 + }, + { + "KL/chosen_KL_mean": -72.64091491699219, + "KL/mean": -101.92864227294922, + "KL/rejected_KL_mean": -131.21636962890625, + "KL/std": 61.07588577270508, + "epoch": 0.6696900982615268, + "fcm_dpo/beta": 0.01461905799806118, + "fcm_dpo/delta": -0.46024078130722046, + "fcm_dpo/margin": 58.5754508972168, + "fcm_dpo/q_t": 0.32659828662872314, + "grad_norm": 16.305545806884766, + "learning_rate": 1.4976736614834662e-07, + "logits/chosen": 0.6411738991737366, + "logits/rejected": 0.5681331753730774, + "logps/chosen": -127.43701934814453, + "logps/ref_chosen": -54.79610061645508, + "logps/ref_rejected": -77.80781555175781, + "logps/rejected": -209.02420043945312, + "loss": 0.8979, + "margin_dpo/margin_mean": 58.57544708251953, + "margin_dpo/margin_std": 66.66512298583984, + "step": 443 + }, + { + "KL/chosen_KL_mean": -92.98896789550781, + "KL/mean": -103.74515533447266, + "KL/rejected_KL_mean": -114.5013427734375, + "KL/std": 58.199806213378906, + "epoch": 0.671201814058957, + "fcm_dpo/beta": 0.014579062350094318, + "fcm_dpo/delta": -0.019250785931944847, + "fcm_dpo/margin": 21.51237678527832, + "fcm_dpo/q_t": 0.434638649225235, + "grad_norm": 22.116947174072266, + "learning_rate": 1.4855747752871654e-07, + "logits/chosen": 0.6510124206542969, + "logits/rejected": 0.5539190769195557, + "logps/chosen": -151.738037109375, + "logps/ref_chosen": -58.749061584472656, + "logps/ref_rejected": -86.87396240234375, + "logps/rejected": -201.37530517578125, + "loss": 1.2786, + "margin_dpo/margin_mean": 21.51237678527832, + "margin_dpo/margin_std": 60.76527404785156, + "step": 444 + }, + { + "KL/chosen_KL_mean": -87.6014404296875, + "KL/mean": -112.07501220703125, + "KL/rejected_KL_mean": -136.548583984375, + "KL/std": 58.561195373535156, + "epoch": 0.672713529856387, + "fcm_dpo/beta": 0.014514950104057789, + "fcm_dpo/delta": -0.3126163184642792, + "fcm_dpo/margin": 48.9471549987793, + "fcm_dpo/q_t": 0.3535197675228119, + "grad_norm": 16.23349380493164, + "learning_rate": 1.473504264745062e-07, + "logits/chosen": 0.6136295199394226, + "logits/rejected": 0.600320041179657, + "logps/chosen": -148.5188751220703, + "logps/ref_chosen": -60.91743850708008, + "logps/ref_rejected": -71.5637435913086, + "logps/rejected": -208.11233520507812, + "loss": 0.9655, + "margin_dpo/margin_mean": 48.9471549987793, + "margin_dpo/margin_std": 60.9539794921875, + "step": 445 + }, + { + "KL/chosen_KL_mean": -76.23194885253906, + "KL/mean": -103.92715454101562, + "KL/rejected_KL_mean": -131.62234497070312, + "KL/std": 59.07441711425781, + "epoch": 0.674225245653817, + "fcm_dpo/beta": 0.014420637860894203, + "fcm_dpo/delta": -0.40179771184921265, + "fcm_dpo/margin": 55.3903923034668, + "fcm_dpo/q_t": 0.33391863107681274, + "grad_norm": 12.375032424926758, + "learning_rate": 1.461462467495284e-07, + "logits/chosen": 0.7065454721450806, + "logits/rejected": 0.6207322478294373, + "logps/chosen": -125.03120422363281, + "logps/ref_chosen": -48.79924774169922, + "logps/ref_rejected": -71.8719482421875, + "logps/rejected": -203.49429321289062, + "loss": 0.8756, + "margin_dpo/margin_mean": 55.3903923034668, + "margin_dpo/margin_std": 52.731178283691406, + "step": 446 + }, + { + "KL/chosen_KL_mean": -73.86038970947266, + "KL/mean": -103.24805450439453, + "KL/rejected_KL_mean": -132.63571166992188, + "KL/std": 58.15815734863281, + "epoch": 0.6757369614512472, + "fcm_dpo/beta": 0.014272423461079597, + "fcm_dpo/delta": -0.4427521228790283, + "fcm_dpo/margin": 58.77533721923828, + "fcm_dpo/q_t": 0.3268117904663086, + "grad_norm": 16.21988868713379, + "learning_rate": 1.4494497203727843e-07, + "logits/chosen": 0.6075619459152222, + "logits/rejected": 0.49892306327819824, + "logps/chosen": -127.54310607910156, + "logps/ref_chosen": -53.682716369628906, + "logps/ref_rejected": -88.17315673828125, + "logps/rejected": -220.8088836669922, + "loss": 0.91, + "margin_dpo/margin_mean": 58.77532958984375, + "margin_dpo/margin_std": 66.51582336425781, + "step": 447 + }, + { + "KL/chosen_KL_mean": -81.37882995605469, + "KL/mean": -104.68525695800781, + "KL/rejected_KL_mean": -127.99169158935547, + "KL/std": 58.45793151855469, + "epoch": 0.6772486772486772, + "fcm_dpo/beta": 0.014189641922712326, + "fcm_dpo/delta": -0.26315754652023315, + "fcm_dpo/margin": 46.61286544799805, + "fcm_dpo/q_t": 0.36013171076774597, + "grad_norm": 13.570533752441406, + "learning_rate": 1.4374663593999256e-07, + "logits/chosen": 0.6740535497665405, + "logits/rejected": 0.6169406175613403, + "logps/chosen": -135.13009643554688, + "logps/ref_chosen": -53.75125503540039, + "logps/ref_rejected": -77.17623901367188, + "logps/rejected": -205.16793823242188, + "loss": 0.9917, + "margin_dpo/margin_mean": 46.61286163330078, + "margin_dpo/margin_std": 60.785675048828125, + "step": 448 + }, + { + "KL/chosen_KL_mean": -99.65214538574219, + "KL/mean": -111.73096466064453, + "KL/rejected_KL_mean": -123.80978393554688, + "KL/std": 58.888160705566406, + "epoch": 0.6787603930461074, + "fcm_dpo/beta": 0.014159944839775562, + "fcm_dpo/delta": -0.038140106946229935, + "fcm_dpo/margin": 24.15762710571289, + "fcm_dpo/q_t": 0.42702996730804443, + "grad_norm": 21.425304412841797, + "learning_rate": 1.4255127197770707e-07, + "logits/chosen": 0.48605477809906006, + "logits/rejected": 0.4832706153392792, + "logps/chosen": -175.47952270507812, + "logps/ref_chosen": -75.82737731933594, + "logps/ref_rejected": -82.20687866210938, + "logps/rejected": -206.01666259765625, + "loss": 1.2066, + "margin_dpo/margin_mean": 24.15762710571289, + "margin_dpo/margin_std": 53.27085494995117, + "step": 449 + }, + { + "KL/chosen_KL_mean": -79.93557739257812, + "KL/mean": -96.75975036621094, + "KL/rejected_KL_mean": -113.58393096923828, + "KL/std": 56.20372772216797, + "epoch": 0.6802721088435374, + "fcm_dpo/beta": 0.014138461090624332, + "fcm_dpo/delta": -0.07615508139133453, + "fcm_dpo/margin": 33.648345947265625, + "fcm_dpo/q_t": 0.40393465757369995, + "grad_norm": 16.800918579101562, + "learning_rate": 1.4135891358732205e-07, + "logits/chosen": 0.7582254409790039, + "logits/rejected": 0.635672926902771, + "logps/chosen": -127.05130004882812, + "logps/ref_chosen": -47.11572265625, + "logps/ref_rejected": -78.7546615600586, + "logps/rejected": -192.33859252929688, + "loss": 1.1377, + "margin_dpo/margin_mean": 33.64834976196289, + "margin_dpo/margin_std": 62.440673828125, + "step": 450 + }, + { + "KL/chosen_KL_mean": -80.29872131347656, + "KL/mean": -97.95948791503906, + "KL/rejected_KL_mean": -115.62025451660156, + "KL/std": 57.17652893066406, + "epoch": 0.6817838246409675, + "fcm_dpo/beta": 0.014120226725935936, + "fcm_dpo/delta": -0.09923126548528671, + "fcm_dpo/margin": 35.32154083251953, + "fcm_dpo/q_t": 0.39510515332221985, + "grad_norm": 16.79901695251465, + "learning_rate": 1.4016959412166437e-07, + "logits/chosen": 0.5782396793365479, + "logits/rejected": 0.5247559547424316, + "logps/chosen": -143.649169921875, + "logps/ref_chosen": -63.350440979003906, + "logps/ref_rejected": -76.28530883789062, + "logps/rejected": -191.9055633544922, + "loss": 1.1002, + "margin_dpo/margin_mean": 35.321537017822266, + "margin_dpo/margin_std": 58.638427734375, + "step": 451 + }, + { + "KL/chosen_KL_mean": -81.18424987792969, + "KL/mean": -100.79125213623047, + "KL/rejected_KL_mean": -120.39823913574219, + "KL/std": 55.176246643066406, + "epoch": 0.6832955404383976, + "fcm_dpo/beta": 0.014078440144658089, + "fcm_dpo/delta": -0.1529121696949005, + "fcm_dpo/margin": 39.213993072509766, + "fcm_dpo/q_t": 0.3839055895805359, + "grad_norm": 16.38026237487793, + "learning_rate": 1.3898334684855645e-07, + "logits/chosen": 0.6040600538253784, + "logits/rejected": 0.5180827975273132, + "logps/chosen": -136.77008056640625, + "logps/ref_chosen": -55.58583450317383, + "logps/ref_rejected": -77.68738555908203, + "logps/rejected": -198.08563232421875, + "loss": 1.0921, + "margin_dpo/margin_mean": 39.21399688720703, + "margin_dpo/margin_std": 64.29176330566406, + "step": 452 + }, + { + "KL/chosen_KL_mean": -77.03958129882812, + "KL/mean": -99.42143249511719, + "KL/rejected_KL_mean": -121.80328369140625, + "KL/std": 57.91150665283203, + "epoch": 0.6848072562358276, + "fcm_dpo/beta": 0.014016878791153431, + "fcm_dpo/delta": -0.22892522811889648, + "fcm_dpo/margin": 44.76371765136719, + "fcm_dpo/q_t": 0.37292051315307617, + "grad_norm": 19.189069747924805, + "learning_rate": 1.3780020494988445e-07, + "logits/chosen": 0.6161945462226868, + "logits/rejected": 0.5839414000511169, + "logps/chosen": -138.81777954101562, + "logps/ref_chosen": -61.778202056884766, + "logps/ref_rejected": -71.51403045654297, + "logps/rejected": -193.31732177734375, + "loss": 1.0336, + "margin_dpo/margin_mean": 44.76371765136719, + "margin_dpo/margin_std": 64.74624633789062, + "step": 453 + }, + { + "KL/chosen_KL_mean": -74.06381225585938, + "KL/mean": -96.71676635742188, + "KL/rejected_KL_mean": -119.36972045898438, + "KL/std": 59.78915786743164, + "epoch": 0.6863189720332578, + "fcm_dpo/beta": 0.013953124172985554, + "fcm_dpo/delta": -0.23365271091461182, + "fcm_dpo/margin": 45.30589294433594, + "fcm_dpo/q_t": 0.36640772223472595, + "grad_norm": 13.258718490600586, + "learning_rate": 1.366202015206706e-07, + "logits/chosen": 0.6213289499282837, + "logits/rejected": 0.5783869028091431, + "logps/chosen": -125.65896606445312, + "logps/ref_chosen": -51.59515380859375, + "logps/ref_rejected": -63.96732711791992, + "logps/rejected": -183.3370361328125, + "loss": 1.0329, + "margin_dpo/margin_mean": 45.30589294433594, + "margin_dpo/margin_std": 65.86431884765625, + "step": 454 + }, + { + "KL/chosen_KL_mean": -82.82814025878906, + "KL/mean": -105.67292785644531, + "KL/rejected_KL_mean": -128.51773071289062, + "KL/std": 57.54645538330078, + "epoch": 0.6878306878306878, + "fcm_dpo/beta": 0.013883685693144798, + "fcm_dpo/delta": -0.23594310879707336, + "fcm_dpo/margin": 45.68959045410156, + "fcm_dpo/q_t": 0.3691120147705078, + "grad_norm": 15.355476379394531, + "learning_rate": 1.354433695681474e-07, + "logits/chosen": 0.513628363609314, + "logits/rejected": 0.47786855697631836, + "logps/chosen": -153.47984313964844, + "logps/ref_chosen": -70.65170288085938, + "logps/ref_rejected": -77.44276428222656, + "logps/rejected": -205.9604949951172, + "loss": 1.0004, + "margin_dpo/margin_mean": 45.68959045410156, + "margin_dpo/margin_std": 60.58068084716797, + "step": 455 + }, + { + "KL/chosen_KL_mean": -82.51515197753906, + "KL/mean": -102.65654754638672, + "KL/rejected_KL_mean": -122.79794311523438, + "KL/std": 56.98590850830078, + "epoch": 0.6893424036281179, + "fcm_dpo/beta": 0.013834391720592976, + "fcm_dpo/delta": -0.15824994444847107, + "fcm_dpo/margin": 40.282798767089844, + "fcm_dpo/q_t": 0.38299810886383057, + "grad_norm": 18.209184646606445, + "learning_rate": 1.3426974201083439e-07, + "logits/chosen": 0.5532491207122803, + "logits/rejected": 0.486372709274292, + "logps/chosen": -138.91343688964844, + "logps/ref_chosen": -56.398284912109375, + "logps/ref_rejected": -82.61642456054688, + "logps/rejected": -205.41436767578125, + "loss": 1.0521, + "margin_dpo/margin_mean": 40.282798767089844, + "margin_dpo/margin_std": 59.3468132019043, + "step": 456 + }, + { + "KL/chosen_KL_mean": -84.92625427246094, + "KL/mean": -105.2531967163086, + "KL/rejected_KL_mean": -125.58013916015625, + "KL/std": 59.5147705078125, + "epoch": 0.690854119425548, + "fcm_dpo/beta": 0.013786647468805313, + "fcm_dpo/delta": -0.16155201196670532, + "fcm_dpo/margin": 40.65387725830078, + "fcm_dpo/q_t": 0.37804293632507324, + "grad_norm": 13.639833450317383, + "learning_rate": 1.3309935167761717e-07, + "logits/chosen": 0.6998741030693054, + "logits/rejected": 0.6211817264556885, + "logps/chosen": -129.64683532714844, + "logps/ref_chosen": -44.72057342529297, + "logps/ref_rejected": -68.1158676147461, + "logps/rejected": -193.6959991455078, + "loss": 1.0211, + "margin_dpo/margin_mean": 40.65387725830078, + "margin_dpo/margin_std": 52.33448791503906, + "step": 457 + }, + { + "KL/chosen_KL_mean": -77.37499237060547, + "KL/mean": -99.87454223632812, + "KL/rejected_KL_mean": -122.37409210205078, + "KL/std": 60.934349060058594, + "epoch": 0.6923658352229781, + "fcm_dpo/beta": 0.013736474327743053, + "fcm_dpo/delta": -0.2195204794406891, + "fcm_dpo/margin": 44.999088287353516, + "fcm_dpo/q_t": 0.3686787486076355, + "grad_norm": 13.035124778747559, + "learning_rate": 1.3193223130682936e-07, + "logits/chosen": 0.6501206159591675, + "logits/rejected": 0.5268113613128662, + "logps/chosen": -127.38068389892578, + "logps/ref_chosen": -50.00569152832031, + "logps/ref_rejected": -87.50015258789062, + "logps/rejected": -209.87423706054688, + "loss": 1.0193, + "margin_dpo/margin_mean": 44.999088287353516, + "margin_dpo/margin_std": 62.240692138671875, + "step": 458 + }, + { + "KL/chosen_KL_mean": -73.94944763183594, + "KL/mean": -101.42481994628906, + "KL/rejected_KL_mean": -128.9001922607422, + "KL/std": 64.1334228515625, + "epoch": 0.6938775510204082, + "fcm_dpo/beta": 0.013669133186340332, + "fcm_dpo/delta": -0.3536731004714966, + "fcm_dpo/margin": 54.95075607299805, + "fcm_dpo/q_t": 0.34350621700286865, + "grad_norm": 16.067678451538086, + "learning_rate": 1.3076841354533658e-07, + "logits/chosen": 0.6525992155075073, + "logits/rejected": 0.6143655776977539, + "logps/chosen": -139.327392578125, + "logps/ref_chosen": -65.37794494628906, + "logps/ref_rejected": -88.19244384765625, + "logps/rejected": -217.0926513671875, + "loss": 0.9383, + "margin_dpo/margin_mean": 54.95075607299805, + "margin_dpo/margin_std": 61.56705856323242, + "step": 459 + }, + { + "KL/chosen_KL_mean": -81.91912078857422, + "KL/mean": -109.09263610839844, + "KL/rejected_KL_mean": -136.26614379882812, + "KL/std": 60.71876525878906, + "epoch": 0.6953892668178382, + "fcm_dpo/beta": 0.013551505282521248, + "fcm_dpo/delta": -0.3391699492931366, + "fcm_dpo/margin": 54.347023010253906, + "fcm_dpo/q_t": 0.34828123450279236, + "grad_norm": 14.13100814819336, + "learning_rate": 1.2960793094762345e-07, + "logits/chosen": 0.6551488637924194, + "logits/rejected": 0.525404691696167, + "logps/chosen": -146.4807891845703, + "logps/ref_chosen": -64.5616683959961, + "logps/ref_rejected": -88.67890167236328, + "logps/rejected": -224.94503784179688, + "loss": 0.9292, + "margin_dpo/margin_mean": 54.347023010253906, + "margin_dpo/margin_std": 61.8521728515625, + "step": 460 + }, + { + "KL/chosen_KL_mean": -67.30126953125, + "KL/mean": -94.51637268066406, + "KL/rejected_KL_mean": -121.73148345947266, + "KL/std": 56.68317413330078, + "epoch": 0.6969009826152683, + "fcm_dpo/beta": 0.013451006263494492, + "fcm_dpo/delta": -0.3352503776550293, + "fcm_dpo/margin": 54.430198669433594, + "fcm_dpo/q_t": 0.3472105860710144, + "grad_norm": 12.946837425231934, + "learning_rate": 1.2845081597488286e-07, + "logits/chosen": 0.7796132564544678, + "logits/rejected": 0.6842553615570068, + "logps/chosen": -116.77919006347656, + "logps/ref_chosen": -49.4779167175293, + "logps/ref_rejected": -72.65262603759766, + "logps/rejected": -194.3841094970703, + "loss": 0.9244, + "margin_dpo/margin_mean": 54.430198669433594, + "margin_dpo/margin_std": 57.968772888183594, + "step": 461 + }, + { + "KL/chosen_KL_mean": -75.794677734375, + "KL/mean": -105.38888549804688, + "KL/rejected_KL_mean": -134.98309326171875, + "KL/std": 59.83213806152344, + "epoch": 0.6984126984126984, + "fcm_dpo/beta": 0.013357922434806824, + "fcm_dpo/delta": -0.39400720596313477, + "fcm_dpo/margin": 59.18840026855469, + "fcm_dpo/q_t": 0.3333611488342285, + "grad_norm": 12.844746589660645, + "learning_rate": 1.27297100994108e-07, + "logits/chosen": 0.6330656409263611, + "logits/rejected": 0.5730718374252319, + "logps/chosen": -136.289794921875, + "logps/ref_chosen": -60.4951171875, + "logps/ref_rejected": -74.82136535644531, + "logps/rejected": -209.804443359375, + "loss": 0.8943, + "margin_dpo/margin_mean": 59.18840026855469, + "margin_dpo/margin_std": 61.932342529296875, + "step": 462 + }, + { + "KL/chosen_KL_mean": -88.986572265625, + "KL/mean": -106.88300323486328, + "KL/rejected_KL_mean": -124.7794418334961, + "KL/std": 56.709136962890625, + "epoch": 0.6999244142101285, + "fcm_dpo/beta": 0.013316287659108639, + "fcm_dpo/delta": -0.07708629965782166, + "fcm_dpo/margin": 35.792869567871094, + "fcm_dpo/q_t": 0.39503830671310425, + "grad_norm": 18.492843627929688, + "learning_rate": 1.2614681827718695e-07, + "logits/chosen": 0.6356483697891235, + "logits/rejected": 0.6348425149917603, + "logps/chosen": -156.67169189453125, + "logps/ref_chosen": -67.68511962890625, + "logps/ref_rejected": -71.32196044921875, + "logps/rejected": -196.1013946533203, + "loss": 1.0918, + "margin_dpo/margin_mean": 35.792869567871094, + "margin_dpo/margin_std": 54.717979431152344, + "step": 463 + }, + { + "KL/chosen_KL_mean": -85.0597152709961, + "KL/mean": -110.97505187988281, + "KL/rejected_KL_mean": -136.890380859375, + "KL/std": 61.78912353515625, + "epoch": 0.7014361300075586, + "fcm_dpo/beta": 0.013265394605696201, + "fcm_dpo/delta": -0.289463609457016, + "fcm_dpo/margin": 51.83067321777344, + "fcm_dpo/q_t": 0.35756415128707886, + "grad_norm": 15.349041938781738, + "learning_rate": 1.2500000000000005e-07, + "logits/chosen": 0.6225231289863586, + "logits/rejected": 0.5926589965820312, + "logps/chosen": -144.22535705566406, + "logps/ref_chosen": -59.16564178466797, + "logps/ref_rejected": -69.56146240234375, + "logps/rejected": -206.45184326171875, + "loss": 0.99, + "margin_dpo/margin_mean": 51.83067321777344, + "margin_dpo/margin_std": 66.81782531738281, + "step": 464 + }, + { + "KL/chosen_KL_mean": -90.03267669677734, + "KL/mean": -110.72211456298828, + "KL/rejected_KL_mean": -131.4115447998047, + "KL/std": 59.351470947265625, + "epoch": 0.7029478458049887, + "fcm_dpo/beta": 0.013209119439125061, + "fcm_dpo/delta": -0.14742261171340942, + "fcm_dpo/margin": 41.378875732421875, + "fcm_dpo/q_t": 0.38601210713386536, + "grad_norm": 18.324115753173828, + "learning_rate": 1.238566782415197e-07, + "logits/chosen": 0.7088480591773987, + "logits/rejected": 0.6419366598129272, + "logps/chosen": -148.54635620117188, + "logps/ref_chosen": -58.513671875, + "logps/ref_rejected": -84.31745910644531, + "logps/rejected": -215.72900390625, + "loss": 1.0818, + "margin_dpo/margin_mean": 41.378875732421875, + "margin_dpo/margin_std": 65.86174774169922, + "step": 465 + }, + { + "KL/chosen_KL_mean": -94.50120544433594, + "KL/mean": -108.36875915527344, + "KL/rejected_KL_mean": -122.23631286621094, + "KL/std": 62.668190002441406, + "epoch": 0.7044595616024187, + "fcm_dpo/beta": 0.013203665614128113, + "fcm_dpo/delta": 0.03390232473611832, + "fcm_dpo/margin": 27.735103607177734, + "fcm_dpo/q_t": 0.4220975637435913, + "grad_norm": 19.471853256225586, + "learning_rate": 1.2271688498291334e-07, + "logits/chosen": 0.6437486410140991, + "logits/rejected": 0.6507591009140015, + "logps/chosen": -167.76702880859375, + "logps/ref_chosen": -73.26580810546875, + "logps/ref_rejected": -74.83621215820312, + "logps/rejected": -197.07252502441406, + "loss": 1.1871, + "margin_dpo/margin_mean": 27.735105514526367, + "margin_dpo/margin_std": 58.35826110839844, + "step": 466 + }, + { + "KL/chosen_KL_mean": -87.67436218261719, + "KL/mean": -107.08904266357422, + "KL/rejected_KL_mean": -126.50371551513672, + "KL/std": 61.65521240234375, + "epoch": 0.7059712773998488, + "fcm_dpo/beta": 0.013187771663069725, + "fcm_dpo/delta": -0.11265860497951508, + "fcm_dpo/margin": 38.829345703125, + "fcm_dpo/q_t": 0.3903040885925293, + "grad_norm": 14.744203567504883, + "learning_rate": 1.2158065210664848e-07, + "logits/chosen": 0.7063431739807129, + "logits/rejected": 0.5445370674133301, + "logps/chosen": -135.25384521484375, + "logps/ref_chosen": -47.57947540283203, + "logps/ref_rejected": -78.68522644042969, + "logps/rejected": -205.18893432617188, + "loss": 1.0582, + "margin_dpo/margin_mean": 38.829349517822266, + "margin_dpo/margin_std": 56.949668884277344, + "step": 467 + }, + { + "KL/chosen_KL_mean": -77.51008605957031, + "KL/mean": -107.94609069824219, + "KL/rejected_KL_mean": -138.38211059570312, + "KL/std": 62.60813903808594, + "epoch": 0.7074829931972789, + "fcm_dpo/beta": 0.013106461614370346, + "fcm_dpo/delta": -0.40091097354888916, + "fcm_dpo/margin": 60.87202453613281, + "fcm_dpo/q_t": 0.33387264609336853, + "grad_norm": 15.976125717163086, + "learning_rate": 1.204480113956011e-07, + "logits/chosen": 0.6125254034996033, + "logits/rejected": 0.6037804484367371, + "logps/chosen": -141.4378662109375, + "logps/ref_chosen": -63.92778778076172, + "logps/ref_rejected": -76.51626586914062, + "logps/rejected": -214.89837646484375, + "loss": 0.893, + "margin_dpo/margin_mean": 60.87202453613281, + "margin_dpo/margin_std": 64.39067077636719, + "step": 468 + }, + { + "KL/chosen_KL_mean": -80.92222595214844, + "KL/mean": -106.03935241699219, + "KL/rejected_KL_mean": -131.15647888183594, + "KL/std": 59.199798583984375, + "epoch": 0.708994708994709, + "fcm_dpo/beta": 0.013013675808906555, + "fcm_dpo/delta": -0.25582316517829895, + "fcm_dpo/margin": 50.2342529296875, + "fcm_dpo/q_t": 0.35991889238357544, + "grad_norm": 17.01838493347168, + "learning_rate": 1.1931899453216697e-07, + "logits/chosen": 0.6833248734474182, + "logits/rejected": 0.6719892024993896, + "logps/chosen": -139.98040771484375, + "logps/ref_chosen": -59.05818176269531, + "logps/ref_rejected": -75.67672729492188, + "logps/rejected": -206.83322143554688, + "loss": 0.9581, + "margin_dpo/margin_mean": 50.2342529296875, + "margin_dpo/margin_std": 55.737796783447266, + "step": 469 + }, + { + "KL/chosen_KL_mean": -81.9334716796875, + "KL/mean": -106.04447937011719, + "KL/rejected_KL_mean": -130.15548706054688, + "KL/std": 58.55548095703125, + "epoch": 0.7105064247921391, + "fcm_dpo/beta": 0.01297076791524887, + "fcm_dpo/delta": -0.2268352210521698, + "fcm_dpo/margin": 48.222023010253906, + "fcm_dpo/q_t": 0.3664366602897644, + "grad_norm": 14.034872055053711, + "learning_rate": 1.1819363309737438e-07, + "logits/chosen": 0.6549979448318481, + "logits/rejected": 0.5794575214385986, + "logps/chosen": -129.8009033203125, + "logps/ref_chosen": -47.86743927001953, + "logps/ref_rejected": -65.96859741210938, + "logps/rejected": -196.12408447265625, + "loss": 1.0105, + "margin_dpo/margin_mean": 48.22202682495117, + "margin_dpo/margin_std": 64.16961669921875, + "step": 470 + }, + { + "KL/chosen_KL_mean": -75.11283111572266, + "KL/mean": -103.60452270507812, + "KL/rejected_KL_mean": -132.09622192382812, + "KL/std": 60.32876205444336, + "epoch": 0.7120181405895691, + "fcm_dpo/beta": 0.012885481119155884, + "fcm_dpo/delta": -0.33672481775283813, + "fcm_dpo/margin": 56.98339080810547, + "fcm_dpo/q_t": 0.34611284732818604, + "grad_norm": 14.195216178894043, + "learning_rate": 1.1707195857000215e-07, + "logits/chosen": 0.6444224119186401, + "logits/rejected": 0.585011899471283, + "logps/chosen": -132.89068603515625, + "logps/ref_chosen": -57.777854919433594, + "logps/ref_rejected": -73.81172180175781, + "logps/rejected": -205.90792846679688, + "loss": 0.942, + "margin_dpo/margin_mean": 56.98339080810547, + "margin_dpo/margin_std": 66.69658660888672, + "step": 471 + }, + { + "KL/chosen_KL_mean": -77.45169830322266, + "KL/mean": -100.25393676757812, + "KL/rejected_KL_mean": -123.05618286132812, + "KL/std": 60.308631896972656, + "epoch": 0.7135298563869993, + "fcm_dpo/beta": 0.012834163382649422, + "fcm_dpo/delta": -0.18634338676929474, + "fcm_dpo/margin": 45.6044807434082, + "fcm_dpo/q_t": 0.38049477338790894, + "grad_norm": 15.8268461227417, + "learning_rate": 1.1595400232569768e-07, + "logits/chosen": 0.6547919511795044, + "logits/rejected": 0.6067420840263367, + "logps/chosen": -133.36036682128906, + "logps/ref_chosen": -55.908668518066406, + "logps/ref_rejected": -74.70294189453125, + "logps/rejected": -197.75912475585938, + "loss": 1.0771, + "margin_dpo/margin_mean": 45.60447692871094, + "margin_dpo/margin_std": 73.08457946777344, + "step": 472 + }, + { + "KL/chosen_KL_mean": -77.85853576660156, + "KL/mean": -101.17366790771484, + "KL/rejected_KL_mean": -124.48878479003906, + "KL/std": 64.7419204711914, + "epoch": 0.7150415721844293, + "fcm_dpo/beta": 0.012776783667504787, + "fcm_dpo/delta": -0.19697603583335876, + "fcm_dpo/margin": 46.63025665283203, + "fcm_dpo/q_t": 0.3795861601829529, + "grad_norm": 17.569110870361328, + "learning_rate": 1.1483979563610069e-07, + "logits/chosen": 0.7309643030166626, + "logits/rejected": 0.6096280813217163, + "logps/chosen": -132.01942443847656, + "logps/ref_chosen": -54.16088104248047, + "logps/ref_rejected": -92.76789855957031, + "logps/rejected": -217.25668334960938, + "loss": 1.0828, + "margin_dpo/margin_mean": 46.63025665283203, + "margin_dpo/margin_std": 76.22259521484375, + "step": 473 + }, + { + "KL/chosen_KL_mean": -87.22453308105469, + "KL/mean": -106.53562927246094, + "KL/rejected_KL_mean": -125.84672546386719, + "KL/std": 62.23670959472656, + "epoch": 0.7165532879818595, + "fcm_dpo/beta": 0.012748065404593945, + "fcm_dpo/delta": -0.09281144291162491, + "fcm_dpo/margin": 38.6221923828125, + "fcm_dpo/q_t": 0.39821857213974, + "grad_norm": 22.02532386779785, + "learning_rate": 1.1372936966796709e-07, + "logits/chosen": 0.7327630519866943, + "logits/rejected": 0.6492011547088623, + "logps/chosen": -133.91024780273438, + "logps/ref_chosen": -46.685707092285156, + "logps/ref_rejected": -71.44731903076172, + "logps/rejected": -197.29403686523438, + "loss": 1.1351, + "margin_dpo/margin_mean": 38.622196197509766, + "margin_dpo/margin_std": 71.04498291015625, + "step": 474 + }, + { + "KL/chosen_KL_mean": -80.06514739990234, + "KL/mean": -112.00347900390625, + "KL/rejected_KL_mean": -143.94180297851562, + "KL/std": 62.18982696533203, + "epoch": 0.7180650037792895, + "fcm_dpo/beta": 0.012656323611736298, + "fcm_dpo/delta": -0.412054181098938, + "fcm_dpo/margin": 63.87665557861328, + "fcm_dpo/q_t": 0.3316466510295868, + "grad_norm": 11.48277759552002, + "learning_rate": 1.126227554822985e-07, + "logits/chosen": 0.64610356092453, + "logits/rejected": 0.5927214622497559, + "logps/chosen": -138.55245971679688, + "logps/ref_chosen": -58.4873046875, + "logps/ref_rejected": -87.00187683105469, + "logps/rejected": -230.94369506835938, + "loss": 0.8722, + "margin_dpo/margin_mean": 63.87665557861328, + "margin_dpo/margin_std": 63.53888702392578, + "step": 475 + }, + { + "KL/chosen_KL_mean": -96.51119995117188, + "KL/mean": -117.64262390136719, + "KL/rejected_KL_mean": -138.7740478515625, + "KL/std": 63.33106231689453, + "epoch": 0.7195767195767195, + "fcm_dpo/beta": 0.012616002932190895, + "fcm_dpo/delta": -0.1338798701763153, + "fcm_dpo/margin": 42.26285171508789, + "fcm_dpo/q_t": 0.3884836435317993, + "grad_norm": 15.622908592224121, + "learning_rate": 1.1151998403347243e-07, + "logits/chosen": 0.5459903478622437, + "logits/rejected": 0.5436645150184631, + "logps/chosen": -171.892822265625, + "logps/ref_chosen": -75.38162231445312, + "logps/ref_rejected": -76.99822235107422, + "logps/rejected": -215.77227783203125, + "loss": 1.0804, + "margin_dpo/margin_mean": 42.26285171508789, + "margin_dpo/margin_std": 66.85415649414062, + "step": 476 + }, + { + "KL/chosen_KL_mean": -97.39051818847656, + "KL/mean": -118.50433349609375, + "KL/rejected_KL_mean": -139.61813354492188, + "KL/std": 63.06694030761719, + "epoch": 0.7210884353741497, + "fcm_dpo/beta": 0.01257837563753128, + "fcm_dpo/delta": -0.13185447454452515, + "fcm_dpo/margin": 42.22760772705078, + "fcm_dpo/q_t": 0.388813853263855, + "grad_norm": 17.39142608642578, + "learning_rate": 1.1042108616837692e-07, + "logits/chosen": 0.690293550491333, + "logits/rejected": 0.6358621716499329, + "logps/chosen": -158.46389770507812, + "logps/ref_chosen": -61.073387145996094, + "logps/ref_rejected": -81.34375, + "logps/rejected": -220.96188354492188, + "loss": 1.1314, + "margin_dpo/margin_mean": 42.22761154174805, + "margin_dpo/margin_std": 76.84716796875, + "step": 477 + }, + { + "KL/chosen_KL_mean": -85.52493286132812, + "KL/mean": -103.82198333740234, + "KL/rejected_KL_mean": -122.11903381347656, + "KL/std": 59.65315628051758, + "epoch": 0.7226001511715797, + "fcm_dpo/beta": 0.012554802000522614, + "fcm_dpo/delta": -0.059744544327259064, + "fcm_dpo/margin": 36.594093322753906, + "fcm_dpo/q_t": 0.40295809507369995, + "grad_norm": 16.35021209716797, + "learning_rate": 1.0932609262554746e-07, + "logits/chosen": 0.6017496585845947, + "logits/rejected": 0.6100105047225952, + "logps/chosen": -142.6922607421875, + "logps/ref_chosen": -57.16731643676758, + "logps/ref_rejected": -53.30917739868164, + "logps/rejected": -175.42820739746094, + "loss": 1.1648, + "margin_dpo/margin_mean": 36.594093322753906, + "margin_dpo/margin_std": 72.25508117675781, + "step": 478 + }, + { + "KL/chosen_KL_mean": -88.9513931274414, + "KL/mean": -104.89715576171875, + "KL/rejected_KL_mean": -120.84292602539062, + "KL/std": 56.3498649597168, + "epoch": 0.7241118669690099, + "fcm_dpo/beta": 0.012547275051474571, + "fcm_dpo/delta": -0.00022766739130020142, + "fcm_dpo/margin": 31.89154052734375, + "fcm_dpo/q_t": 0.4132658541202545, + "grad_norm": 17.44011878967285, + "learning_rate": 1.0823503403430734e-07, + "logits/chosen": 0.5980551242828369, + "logits/rejected": 0.5462213754653931, + "logps/chosen": -147.86471557617188, + "logps/ref_chosen": -58.91331481933594, + "logps/ref_rejected": -63.7403450012207, + "logps/rejected": -184.58328247070312, + "loss": 1.1702, + "margin_dpo/margin_mean": 31.89154052734375, + "margin_dpo/margin_std": 63.32213592529297, + "step": 479 + }, + { + "KL/chosen_KL_mean": -87.70037078857422, + "KL/mean": -113.26460266113281, + "KL/rejected_KL_mean": -138.82882690429688, + "KL/std": 59.2943229675293, + "epoch": 0.7256235827664399, + "fcm_dpo/beta": 0.01252063550055027, + "fcm_dpo/delta": -0.24165111780166626, + "fcm_dpo/margin": 51.128448486328125, + "fcm_dpo/q_t": 0.36327680945396423, + "grad_norm": 17.7063045501709, + "learning_rate": 1.0714794091391072e-07, + "logits/chosen": 0.585403323173523, + "logits/rejected": 0.57558274269104, + "logps/chosen": -150.5009765625, + "logps/ref_chosen": -62.80061340332031, + "logps/ref_rejected": -67.58859252929688, + "logps/rejected": -206.41741943359375, + "loss": 1.0197, + "margin_dpo/margin_mean": 51.128448486328125, + "margin_dpo/margin_std": 69.74596405029297, + "step": 480 + }, + { + "KL/chosen_KL_mean": -87.1451416015625, + "KL/mean": -111.0331802368164, + "KL/rejected_KL_mean": -134.9212188720703, + "KL/std": 63.15415954589844, + "epoch": 0.72713529856387, + "fcm_dpo/beta": 0.012454831972718239, + "fcm_dpo/delta": -0.19626690447330475, + "fcm_dpo/margin": 47.776092529296875, + "fcm_dpo/q_t": 0.3750731945037842, + "grad_norm": 14.753815650939941, + "learning_rate": 1.0606484367268906e-07, + "logits/chosen": 0.5874171257019043, + "logits/rejected": 0.584084153175354, + "logps/chosen": -152.43162536621094, + "logps/ref_chosen": -65.28649139404297, + "logps/ref_rejected": -70.78668212890625, + "logps/rejected": -205.70791625976562, + "loss": 1.0353, + "margin_dpo/margin_mean": 47.776100158691406, + "margin_dpo/margin_std": 68.3729476928711, + "step": 481 + }, + { + "KL/chosen_KL_mean": -98.50666809082031, + "KL/mean": -120.1825180053711, + "KL/rejected_KL_mean": -141.85836791992188, + "KL/std": 63.53254699707031, + "epoch": 0.7286470143613001, + "fcm_dpo/beta": 0.012423858046531677, + "fcm_dpo/delta": -0.13932153582572937, + "fcm_dpo/margin": 43.35169219970703, + "fcm_dpo/q_t": 0.39148762822151184, + "grad_norm": 18.664758682250977, + "learning_rate": 1.0498577260720048e-07, + "logits/chosen": 0.5445523262023926, + "logits/rejected": 0.3933546543121338, + "logps/chosen": -159.41285705566406, + "logps/ref_chosen": -60.906185150146484, + "logps/ref_rejected": -103.44656372070312, + "logps/rejected": -245.304931640625, + "loss": 1.1362, + "margin_dpo/margin_mean": 43.351688385009766, + "margin_dpo/margin_std": 81.07269287109375, + "step": 482 + }, + { + "KL/chosen_KL_mean": -83.346923828125, + "KL/mean": -109.9117431640625, + "KL/rejected_KL_mean": -136.4765625, + "KL/std": 59.528175354003906, + "epoch": 0.7301587301587301, + "fcm_dpo/beta": 0.01236402802169323, + "fcm_dpo/delta": -0.2585721015930176, + "fcm_dpo/margin": 53.129634857177734, + "fcm_dpo/q_t": 0.35918521881103516, + "grad_norm": 14.820045471191406, + "learning_rate": 1.0391075790138232e-07, + "logits/chosen": 0.7240715622901917, + "logits/rejected": 0.610710859298706, + "logps/chosen": -136.5389404296875, + "logps/ref_chosen": -53.192012786865234, + "logps/ref_rejected": -81.83927154541016, + "logps/rejected": -218.31582641601562, + "loss": 0.9778, + "margin_dpo/margin_mean": 53.129638671875, + "margin_dpo/margin_std": 65.63418579101562, + "step": 483 + }, + { + "KL/chosen_KL_mean": -85.29837036132812, + "KL/mean": -105.71441650390625, + "KL/rejected_KL_mean": -126.13044738769531, + "KL/std": 58.477333068847656, + "epoch": 0.7316704459561603, + "fcm_dpo/beta": 0.012331999838352203, + "fcm_dpo/delta": -0.10405363142490387, + "fcm_dpo/margin": 40.83207702636719, + "fcm_dpo/q_t": 0.3882726728916168, + "grad_norm": 18.47596549987793, + "learning_rate": 1.0283982962570681e-07, + "logits/chosen": 0.7264783978462219, + "logits/rejected": 0.6884661912918091, + "logps/chosen": -143.06784057617188, + "logps/ref_chosen": -57.76945877075195, + "logps/ref_rejected": -71.6829833984375, + "logps/rejected": -197.8134307861328, + "loss": 1.0418, + "margin_dpo/margin_mean": 40.83207321166992, + "margin_dpo/margin_std": 52.934234619140625, + "step": 484 + }, + { + "KL/chosen_KL_mean": -85.53819274902344, + "KL/mean": -106.93986511230469, + "KL/rejected_KL_mean": -128.34152221679688, + "KL/std": 61.29725646972656, + "epoch": 0.7331821617535903, + "fcm_dpo/beta": 0.012286081910133362, + "fcm_dpo/delta": -0.12685120105743408, + "fcm_dpo/margin": 42.803314208984375, + "fcm_dpo/q_t": 0.38722407817840576, + "grad_norm": 15.01928424835205, + "learning_rate": 1.0177301773633992e-07, + "logits/chosen": 0.6463805437088013, + "logits/rejected": 0.6214851140975952, + "logps/chosen": -142.17404174804688, + "logps/ref_chosen": -56.63584899902344, + "logps/ref_rejected": -70.85614013671875, + "logps/rejected": -199.19766235351562, + "loss": 1.0579, + "margin_dpo/margin_mean": 42.803314208984375, + "margin_dpo/margin_std": 60.79389190673828, + "step": 485 + }, + { + "KL/chosen_KL_mean": -104.0463638305664, + "KL/mean": -122.2401123046875, + "KL/rejected_KL_mean": -140.43386840820312, + "KL/std": 66.77265930175781, + "epoch": 0.7346938775510204, + "fcm_dpo/beta": 0.012282265350222588, + "fcm_dpo/delta": -0.047129034996032715, + "fcm_dpo/margin": 36.38750076293945, + "fcm_dpo/q_t": 0.4084394574165344, + "grad_norm": 14.847362518310547, + "learning_rate": 1.007103520743035e-07, + "logits/chosen": 0.68473219871521, + "logits/rejected": 0.5533545017242432, + "logps/chosen": -160.3933868408203, + "logps/ref_chosen": -56.347023010253906, + "logps/ref_rejected": -85.97221374511719, + "logps/rejected": -226.40606689453125, + "loss": 1.1615, + "margin_dpo/margin_mean": 36.38749694824219, + "margin_dpo/margin_std": 72.79695129394531, + "step": 486 + }, + { + "KL/chosen_KL_mean": -93.34505462646484, + "KL/mean": -115.34041595458984, + "KL/rejected_KL_mean": -137.33578491210938, + "KL/std": 58.61061096191406, + "epoch": 0.7362055933484505, + "fcm_dpo/beta": 0.012250418774783611, + "fcm_dpo/delta": -0.1396898329257965, + "fcm_dpo/margin": 43.99073028564453, + "fcm_dpo/q_t": 0.38480663299560547, + "grad_norm": 17.677383422851562, + "learning_rate": 9.965186236464046e-08, + "logits/chosen": 0.7442872524261475, + "logits/rejected": 0.6766370534896851, + "logps/chosen": -153.9622802734375, + "logps/ref_chosen": -60.617218017578125, + "logps/ref_rejected": -82.50975036621094, + "logps/rejected": -219.84552001953125, + "loss": 1.0547, + "margin_dpo/margin_mean": 43.990726470947266, + "margin_dpo/margin_std": 64.96102142333984, + "step": 487 + }, + { + "KL/chosen_KL_mean": -83.74758911132812, + "KL/mean": -109.63170623779297, + "KL/rejected_KL_mean": -135.5158233642578, + "KL/std": 59.41032791137695, + "epoch": 0.7377173091458806, + "fcm_dpo/beta": 0.012207914143800735, + "fcm_dpo/delta": -0.23339006304740906, + "fcm_dpo/margin": 51.76823425292969, + "fcm_dpo/q_t": 0.36637309193611145, + "grad_norm": 17.834232330322266, + "learning_rate": 9.859757821558337e-08, + "logits/chosen": 0.649517297744751, + "logits/rejected": 0.580168604850769, + "logps/chosen": -146.8566436767578, + "logps/ref_chosen": -63.10905075073242, + "logps/ref_rejected": -82.49348449707031, + "logps/rejected": -218.00930786132812, + "loss": 0.9812, + "margin_dpo/margin_mean": 51.76823425292969, + "margin_dpo/margin_std": 62.961036682128906, + "step": 488 + }, + { + "KL/chosen_KL_mean": -99.00667572021484, + "KL/mean": -114.16756439208984, + "KL/rejected_KL_mean": -129.32846069335938, + "KL/std": 63.458011627197266, + "epoch": 0.7392290249433107, + "fcm_dpo/beta": 0.012190176174044609, + "fcm_dpo/delta": 0.030482716858386993, + "fcm_dpo/margin": 30.321788787841797, + "fcm_dpo/q_t": 0.421286016702652, + "grad_norm": 16.102081298828125, + "learning_rate": 9.754752911772615e-08, + "logits/chosen": 0.6026809811592102, + "logits/rejected": 0.5534663200378418, + "logps/chosen": -163.99563598632812, + "logps/ref_chosen": -64.98896026611328, + "logps/ref_rejected": -84.39607238769531, + "logps/rejected": -213.72451782226562, + "loss": 1.2049, + "margin_dpo/margin_mean": 30.321788787841797, + "margin_dpo/margin_std": 67.5270767211914, + "step": 489 + }, + { + "KL/chosen_KL_mean": -88.02426147460938, + "KL/mean": -107.21294403076172, + "KL/rejected_KL_mean": -126.40162658691406, + "KL/std": 64.46929931640625, + "epoch": 0.7407407407407407, + "fcm_dpo/beta": 0.01217743568122387, + "fcm_dpo/delta": -0.0677100345492363, + "fcm_dpo/margin": 38.37737274169922, + "fcm_dpo/q_t": 0.4014626145362854, + "grad_norm": 13.697748184204102, + "learning_rate": 9.650174444319956e-08, + "logits/chosen": 0.7481526136398315, + "logits/rejected": 0.7201690077781677, + "logps/chosen": -149.93301391601562, + "logps/ref_chosen": -61.90874481201172, + "logps/ref_rejected": -70.58566284179688, + "logps/rejected": -196.98728942871094, + "loss": 1.1843, + "margin_dpo/margin_mean": 38.37737274169922, + "margin_dpo/margin_std": 79.66912841796875, + "step": 490 + }, + { + "KL/chosen_KL_mean": -87.31063842773438, + "KL/mean": -108.83796691894531, + "KL/rejected_KL_mean": -130.36532592773438, + "KL/std": 59.50713348388672, + "epoch": 0.7422524565381708, + "fcm_dpo/beta": 0.012145858258008957, + "fcm_dpo/delta": -0.1238037645816803, + "fcm_dpo/margin": 43.054664611816406, + "fcm_dpo/q_t": 0.3876134157180786, + "grad_norm": 14.066550254821777, + "learning_rate": 9.546025344484868e-08, + "logits/chosen": 0.5910606384277344, + "logits/rejected": 0.5329212546348572, + "logps/chosen": -142.78634643554688, + "logps/ref_chosen": -55.47570037841797, + "logps/ref_rejected": -78.70318603515625, + "logps/rejected": -209.06849670410156, + "loss": 1.0628, + "margin_dpo/margin_mean": 43.054664611816406, + "margin_dpo/margin_std": 62.32221221923828, + "step": 491 + }, + { + "KL/chosen_KL_mean": -105.6453628540039, + "KL/mean": -122.65867614746094, + "KL/rejected_KL_mean": -139.6719970703125, + "KL/std": 65.59391021728516, + "epoch": 0.7437641723356009, + "fcm_dpo/beta": 0.012152638286352158, + "fcm_dpo/delta": -0.013632431626319885, + "fcm_dpo/margin": 34.02663040161133, + "fcm_dpo/q_t": 0.4130189120769501, + "grad_norm": 18.032333374023438, + "learning_rate": 9.442308525541589e-08, + "logits/chosen": 0.6288784742355347, + "logits/rejected": 0.5509954690933228, + "logps/chosen": -172.93174743652344, + "logps/ref_chosen": -67.28638458251953, + "logps/ref_rejected": -82.78628540039062, + "logps/rejected": -222.45828247070312, + "loss": 1.1868, + "margin_dpo/margin_mean": 34.02663040161133, + "margin_dpo/margin_std": 70.32714080810547, + "step": 492 + }, + { + "KL/chosen_KL_mean": -83.97378540039062, + "KL/mean": -110.05132293701172, + "KL/rejected_KL_mean": -136.12887573242188, + "KL/std": 64.4298324584961, + "epoch": 0.745275888133031, + "fcm_dpo/beta": 0.012111847288906574, + "fcm_dpo/delta": -0.23310865461826324, + "fcm_dpo/margin": 52.15508270263672, + "fcm_dpo/q_t": 0.36435532569885254, + "grad_norm": 13.867661476135254, + "learning_rate": 9.339026888672468e-08, + "logits/chosen": 0.5987674593925476, + "logits/rejected": 0.5121057033538818, + "logps/chosen": -139.9012908935547, + "logps/ref_chosen": -55.92750549316406, + "logps/ref_rejected": -79.12149810791016, + "logps/rejected": -215.2503662109375, + "loss": 0.9972, + "margin_dpo/margin_mean": 52.15509033203125, + "margin_dpo/margin_std": 65.84524536132812, + "step": 493 + }, + { + "KL/chosen_KL_mean": -88.52896118164062, + "KL/mean": -108.56002807617188, + "KL/rejected_KL_mean": -128.59109497070312, + "KL/std": 62.48086929321289, + "epoch": 0.7467876039304611, + "fcm_dpo/beta": 0.012070360593497753, + "fcm_dpo/delta": -0.08399654924869537, + "fcm_dpo/margin": 40.062129974365234, + "fcm_dpo/q_t": 0.3971877098083496, + "grad_norm": 17.83734893798828, + "learning_rate": 9.236183322886945e-08, + "logits/chosen": 0.5516543388366699, + "logits/rejected": 0.492398738861084, + "logps/chosen": -156.48306274414062, + "logps/ref_chosen": -67.95410919189453, + "logps/ref_rejected": -90.50865173339844, + "logps/rejected": -219.09974670410156, + "loss": 1.1512, + "margin_dpo/margin_mean": 40.06212615966797, + "margin_dpo/margin_std": 77.24173736572266, + "step": 494 + }, + { + "KL/chosen_KL_mean": -85.454345703125, + "KL/mean": -102.8785400390625, + "KL/rejected_KL_mean": -120.302734375, + "KL/std": 61.02322769165039, + "epoch": 0.7482993197278912, + "fcm_dpo/beta": 0.012069910764694214, + "fcm_dpo/delta": -0.020741432905197144, + "fcm_dpo/margin": 34.848388671875, + "fcm_dpo/q_t": 0.41301801800727844, + "grad_norm": 16.923980712890625, + "learning_rate": 9.133780704940594e-08, + "logits/chosen": 0.7149494886398315, + "logits/rejected": 0.6408044099807739, + "logps/chosen": -138.07980346679688, + "logps/ref_chosen": -52.62546157836914, + "logps/ref_rejected": -72.06781005859375, + "logps/rejected": -192.37054443359375, + "loss": 1.1556, + "margin_dpo/margin_mean": 34.848392486572266, + "margin_dpo/margin_std": 67.20562744140625, + "step": 495 + }, + { + "KL/chosen_KL_mean": -94.88565063476562, + "KL/mean": -117.21812438964844, + "KL/rejected_KL_mean": -139.55059814453125, + "KL/std": 64.93536376953125, + "epoch": 0.7498110355253212, + "fcm_dpo/beta": 0.012028923258185387, + "fcm_dpo/delta": -0.13822439312934875, + "fcm_dpo/margin": 44.66493225097656, + "fcm_dpo/q_t": 0.39367783069610596, + "grad_norm": 14.428415298461914, + "learning_rate": 9.031821899254797e-08, + "logits/chosen": 0.6608551740646362, + "logits/rejected": 0.5398832559585571, + "logps/chosen": -152.48297119140625, + "logps/ref_chosen": -57.597320556640625, + "logps/ref_rejected": -94.36127471923828, + "logps/rejected": -233.911865234375, + "loss": 1.0979, + "margin_dpo/margin_mean": 44.66493225097656, + "margin_dpo/margin_std": 76.66626739501953, + "step": 496 + }, + { + "KL/chosen_KL_mean": -95.19734191894531, + "KL/mean": -120.67750549316406, + "KL/rejected_KL_mean": -146.1576690673828, + "KL/std": 63.578895568847656, + "epoch": 0.7513227513227513, + "fcm_dpo/beta": 0.011989409103989601, + "fcm_dpo/delta": -0.21231910586357117, + "fcm_dpo/margin": 50.960323333740234, + "fcm_dpo/q_t": 0.3678181767463684, + "grad_norm": 14.277939796447754, + "learning_rate": 8.930309757836516e-08, + "logits/chosen": 0.6894056797027588, + "logits/rejected": 0.6548833250999451, + "logps/chosen": -167.98728942871094, + "logps/ref_chosen": -72.78994750976562, + "logps/ref_rejected": -89.48483276367188, + "logps/rejected": -235.6425018310547, + "loss": 1.0069, + "margin_dpo/margin_mean": 50.96031951904297, + "margin_dpo/margin_std": 66.27978515625, + "step": 497 + }, + { + "KL/chosen_KL_mean": -85.08381652832031, + "KL/mean": -110.19229888916016, + "KL/rejected_KL_mean": -135.30078125, + "KL/std": 61.82719039916992, + "epoch": 0.7528344671201814, + "fcm_dpo/beta": 0.011941779404878616, + "fcm_dpo/delta": -0.20089754462242126, + "fcm_dpo/margin": 50.21696472167969, + "fcm_dpo/q_t": 0.36982592940330505, + "grad_norm": 15.083342552185059, + "learning_rate": 8.829247120198563e-08, + "logits/chosen": 0.6359756588935852, + "logits/rejected": 0.602130651473999, + "logps/chosen": -153.4495391845703, + "logps/ref_chosen": -68.36572265625, + "logps/ref_rejected": -71.28846740722656, + "logps/rejected": -206.58924865722656, + "loss": 0.9909, + "margin_dpo/margin_mean": 50.21696853637695, + "margin_dpo/margin_std": 61.286651611328125, + "step": 498 + }, + { + "KL/chosen_KL_mean": -82.20956420898438, + "KL/mean": -107.16743469238281, + "KL/rejected_KL_mean": -132.12527465820312, + "KL/std": 63.762786865234375, + "epoch": 0.7543461829176115, + "fcm_dpo/beta": 0.011898322030901909, + "fcm_dpo/delta": -0.19503849744796753, + "fcm_dpo/margin": 49.915733337402344, + "fcm_dpo/q_t": 0.37551143765449524, + "grad_norm": 17.06244468688965, + "learning_rate": 8.728636813280163e-08, + "logits/chosen": 0.6331771612167358, + "logits/rejected": 0.561076283454895, + "logps/chosen": -144.11839294433594, + "logps/ref_chosen": -61.90882873535156, + "logps/ref_rejected": -91.9411392211914, + "logps/rejected": -224.06642150878906, + "loss": 1.0704, + "margin_dpo/margin_mean": 49.915733337402344, + "margin_dpo/margin_std": 77.73551177978516, + "step": 499 + }, + { + "KL/chosen_KL_mean": -89.0292739868164, + "KL/mean": -111.50369262695312, + "KL/rejected_KL_mean": -133.97808837890625, + "KL/std": 59.32463073730469, + "epoch": 0.7558578987150416, + "fcm_dpo/beta": 0.011858111247420311, + "fcm_dpo/delta": -0.13375121355056763, + "fcm_dpo/margin": 44.948822021484375, + "fcm_dpo/q_t": 0.38408055901527405, + "grad_norm": 18.058481216430664, + "learning_rate": 8.628481651367875e-08, + "logits/chosen": 0.5656280517578125, + "logits/rejected": 0.5612741708755493, + "logps/chosen": -159.25509643554688, + "logps/ref_chosen": -70.225830078125, + "logps/ref_rejected": -71.72203063964844, + "logps/rejected": -205.70013427734375, + "loss": 1.0802, + "margin_dpo/margin_mean": 44.948822021484375, + "margin_dpo/margin_std": 70.2720947265625, + "step": 500 + }, + { + "KL/chosen_KL_mean": -88.0543441772461, + "KL/mean": -108.39398193359375, + "KL/rejected_KL_mean": -128.73361206054688, + "KL/std": 62.700531005859375, + "epoch": 0.7573696145124716, + "fcm_dpo/beta": 0.011842433363199234, + "fcm_dpo/delta": -0.08213633298873901, + "fcm_dpo/margin": 40.679264068603516, + "fcm_dpo/q_t": 0.39155471324920654, + "grad_norm": 11.885863304138184, + "learning_rate": 8.528784436016878e-08, + "logits/chosen": 0.6474437713623047, + "logits/rejected": 0.6452994346618652, + "logps/chosen": -152.6531524658203, + "logps/ref_chosen": -64.59880828857422, + "logps/ref_rejected": -70.59329223632812, + "logps/rejected": -199.326904296875, + "loss": 1.0455, + "margin_dpo/margin_mean": 40.67926025390625, + "margin_dpo/margin_std": 50.47174835205078, + "step": 501 + }, + { + "KL/chosen_KL_mean": -87.0926513671875, + "KL/mean": -108.92316436767578, + "KL/rejected_KL_mean": -130.753662109375, + "KL/std": 63.958740234375, + "epoch": 0.7588813303099018, + "fcm_dpo/beta": 0.01181572861969471, + "fcm_dpo/delta": -0.11647076904773712, + "fcm_dpo/margin": 43.66101837158203, + "fcm_dpo/q_t": 0.3866531252861023, + "grad_norm": 15.413098335266113, + "learning_rate": 8.4295479559726e-08, + "logits/chosen": 0.6285011172294617, + "logits/rejected": 0.5788655281066895, + "logps/chosen": -152.55929565429688, + "logps/ref_chosen": -65.46662902832031, + "logps/ref_rejected": -90.22233581542969, + "logps/rejected": -220.97601318359375, + "loss": 1.0394, + "margin_dpo/margin_mean": 43.66101837158203, + "margin_dpo/margin_std": 57.39012145996094, + "step": 502 + }, + { + "KL/chosen_KL_mean": -80.45634460449219, + "KL/mean": -102.69293212890625, + "KL/rejected_KL_mean": -124.92951965332031, + "KL/std": 62.399314880371094, + "epoch": 0.7603930461073318, + "fcm_dpo/beta": 0.011780839413404465, + "fcm_dpo/delta": -0.12461839616298676, + "fcm_dpo/margin": 44.473182678222656, + "fcm_dpo/q_t": 0.38616055250167847, + "grad_norm": 13.424242973327637, + "learning_rate": 8.330774987092712e-08, + "logits/chosen": 0.6445924043655396, + "logits/rejected": 0.6430461406707764, + "logps/chosen": -132.29110717773438, + "logps/ref_chosen": -51.83476257324219, + "logps/ref_rejected": -57.62522506713867, + "logps/rejected": -182.55474853515625, + "loss": 1.0834, + "margin_dpo/margin_mean": 44.473182678222656, + "margin_dpo/margin_std": 69.4652099609375, + "step": 503 + }, + { + "KL/chosen_KL_mean": -82.74490356445312, + "KL/mean": -112.27587890625, + "KL/rejected_KL_mean": -141.80685424804688, + "KL/std": 60.623722076416016, + "epoch": 0.7619047619047619, + "fcm_dpo/beta": 0.011732323095202446, + "fcm_dpo/delta": -0.294869601726532, + "fcm_dpo/margin": 59.06193923950195, + "fcm_dpo/q_t": 0.34972602128982544, + "grad_norm": 17.633028030395508, + "learning_rate": 8.232468292269479e-08, + "logits/chosen": 0.5838197469711304, + "logits/rejected": 0.559502899646759, + "logps/chosen": -151.39610290527344, + "logps/ref_chosen": -68.65119934082031, + "logps/ref_rejected": -77.91394805908203, + "logps/rejected": -219.72079467773438, + "loss": 0.9207, + "margin_dpo/margin_mean": 59.06194305419922, + "margin_dpo/margin_std": 59.80674743652344, + "step": 504 + }, + { + "KL/chosen_KL_mean": -86.4732666015625, + "KL/mean": -105.55291748046875, + "KL/rejected_KL_mean": -124.63255310058594, + "KL/std": 66.83235168457031, + "epoch": 0.763416477702192, + "fcm_dpo/beta": 0.011688269674777985, + "fcm_dpo/delta": -0.046417176723480225, + "fcm_dpo/margin": 38.15928268432617, + "fcm_dpo/q_t": 0.40786224603652954, + "grad_norm": 15.572614669799805, + "learning_rate": 8.134630621352483e-08, + "logits/chosen": 0.5979966521263123, + "logits/rejected": 0.5606677532196045, + "logps/chosen": -146.4721221923828, + "logps/ref_chosen": -59.99884796142578, + "logps/ref_rejected": -76.88048553466797, + "logps/rejected": -201.51303100585938, + "loss": 1.1677, + "margin_dpo/margin_mean": 38.159278869628906, + "margin_dpo/margin_std": 74.90796661376953, + "step": 505 + }, + { + "KL/chosen_KL_mean": -83.4315185546875, + "KL/mean": -104.4886703491211, + "KL/rejected_KL_mean": -125.54582214355469, + "KL/std": 59.93007278442383, + "epoch": 0.764928193499622, + "fcm_dpo/beta": 0.011676234193146229, + "fcm_dpo/delta": -0.09223389625549316, + "fcm_dpo/margin": 42.11429977416992, + "fcm_dpo/q_t": 0.39337825775146484, + "grad_norm": 15.781412124633789, + "learning_rate": 8.037264711071698e-08, + "logits/chosen": 0.6514492034912109, + "logits/rejected": 0.6315554976463318, + "logps/chosen": -153.50282287597656, + "logps/ref_chosen": -70.07130432128906, + "logps/ref_rejected": -82.03775024414062, + "logps/rejected": -207.5835723876953, + "loss": 1.1151, + "margin_dpo/margin_mean": 42.11429977416992, + "margin_dpo/margin_std": 72.2458267211914, + "step": 506 + }, + { + "KL/chosen_KL_mean": -93.22354125976562, + "KL/mean": -115.87088775634766, + "KL/rejected_KL_mean": -138.5182342529297, + "KL/std": 64.01895904541016, + "epoch": 0.7664399092970522, + "fcm_dpo/beta": 0.01164855808019638, + "fcm_dpo/delta": -0.1283462941646576, + "fcm_dpo/margin": 45.29468536376953, + "fcm_dpo/q_t": 0.3928554952144623, + "grad_norm": 15.166808128356934, + "learning_rate": 7.940373284960933e-08, + "logits/chosen": 0.6396739482879639, + "logits/rejected": 0.5840606689453125, + "logps/chosen": -165.23057556152344, + "logps/ref_chosen": -72.00703430175781, + "logps/ref_rejected": -93.94987487792969, + "logps/rejected": -232.46810913085938, + "loss": 1.0892, + "margin_dpo/margin_mean": 45.29468536376953, + "margin_dpo/margin_std": 73.7684555053711, + "step": 507 + }, + { + "KL/chosen_KL_mean": -83.59366607666016, + "KL/mean": -110.03814697265625, + "KL/rejected_KL_mean": -136.4826202392578, + "KL/std": 66.6182632446289, + "epoch": 0.7679516250944822, + "fcm_dpo/beta": 0.011613940820097923, + "fcm_dpo/delta": -0.2155165672302246, + "fcm_dpo/margin": 52.888946533203125, + "fcm_dpo/q_t": 0.3692435026168823, + "grad_norm": 15.544646263122559, + "learning_rate": 7.843959053281663e-08, + "logits/chosen": 0.6086411476135254, + "logits/rejected": 0.47358691692352295, + "logps/chosen": -143.8135986328125, + "logps/ref_chosen": -60.21992492675781, + "logps/ref_rejected": -95.9200668334961, + "logps/rejected": -232.40267944335938, + "loss": 1.0065, + "margin_dpo/margin_mean": 52.888946533203125, + "margin_dpo/margin_std": 68.44908142089844, + "step": 508 + }, + { + "KL/chosen_KL_mean": -89.94052124023438, + "KL/mean": -111.10928344726562, + "KL/rejected_KL_mean": -132.2780303955078, + "KL/std": 62.40410232543945, + "epoch": 0.7694633408919124, + "fcm_dpo/beta": 0.011572863906621933, + "fcm_dpo/delta": -0.09050148725509644, + "fcm_dpo/margin": 42.3375129699707, + "fcm_dpo/q_t": 0.39171361923217773, + "grad_norm": 16.44659423828125, + "learning_rate": 7.748024712947204e-08, + "logits/chosen": 0.5528764128684998, + "logits/rejected": 0.5304781794548035, + "logps/chosen": -156.210693359375, + "logps/ref_chosen": -66.27017211914062, + "logps/ref_rejected": -71.73065185546875, + "logps/rejected": -204.00868225097656, + "loss": 1.0832, + "margin_dpo/margin_mean": 42.3375129699707, + "margin_dpo/margin_std": 64.01100158691406, + "step": 509 + }, + { + "KL/chosen_KL_mean": -89.6832275390625, + "KL/mean": -114.82077026367188, + "KL/rejected_KL_mean": -139.95831298828125, + "KL/std": 63.969093322753906, + "epoch": 0.7709750566893424, + "fcm_dpo/beta": 0.011536836624145508, + "fcm_dpo/delta": -0.18118345737457275, + "fcm_dpo/margin": 50.27507019042969, + "fcm_dpo/q_t": 0.37885794043540955, + "grad_norm": 14.513899803161621, + "learning_rate": 7.652572947447272e-08, + "logits/chosen": 0.7556344866752625, + "logits/rejected": 0.6477383971214294, + "logps/chosen": -143.22811889648438, + "logps/ref_chosen": -53.54487609863281, + "logps/ref_rejected": -91.36648559570312, + "logps/rejected": -231.32479858398438, + "loss": 1.0493, + "margin_dpo/margin_mean": 50.27507019042969, + "margin_dpo/margin_std": 74.0670166015625, + "step": 510 + }, + { + "KL/chosen_KL_mean": -87.86951446533203, + "KL/mean": -116.3458251953125, + "KL/rejected_KL_mean": -144.82212829589844, + "KL/std": 62.9260368347168, + "epoch": 0.7724867724867724, + "fcm_dpo/beta": 0.011489283293485641, + "fcm_dpo/delta": -0.2559814453125, + "fcm_dpo/margin": 56.952613830566406, + "fcm_dpo/q_t": 0.3612075746059418, + "grad_norm": 15.84493350982666, + "learning_rate": 7.557606426772961e-08, + "logits/chosen": 0.6756493449211121, + "logits/rejected": 0.6125138998031616, + "logps/chosen": -143.71389770507812, + "logps/ref_chosen": -55.844383239746094, + "logps/ref_rejected": -86.49819946289062, + "logps/rejected": -231.32032775878906, + "loss": 0.9949, + "margin_dpo/margin_mean": 56.952613830566406, + "margin_dpo/margin_std": 73.27781677246094, + "step": 511 + }, + { + "KL/chosen_KL_mean": -86.50758361816406, + "KL/mean": -107.4898681640625, + "KL/rejected_KL_mean": -128.47216796875, + "KL/std": 59.05572509765625, + "epoch": 0.7739984882842026, + "fcm_dpo/beta": 0.01145764160901308, + "fcm_dpo/delta": -0.08121366798877716, + "fcm_dpo/margin": 41.964599609375, + "fcm_dpo/q_t": 0.3966831564903259, + "grad_norm": 17.862192153930664, + "learning_rate": 7.463127807341966e-08, + "logits/chosen": 0.5313820838928223, + "logits/rejected": 0.52196204662323, + "logps/chosen": -148.16061401367188, + "logps/ref_chosen": -61.653038024902344, + "logps/ref_rejected": -72.83148193359375, + "logps/rejected": -201.30364990234375, + "loss": 1.1187, + "margin_dpo/margin_mean": 41.964595794677734, + "margin_dpo/margin_std": 72.26940155029297, + "step": 512 + }, + { + "KL/chosen_KL_mean": -78.30789184570312, + "KL/mean": -101.77067565917969, + "KL/rejected_KL_mean": -125.23347473144531, + "KL/std": 61.8763427734375, + "epoch": 0.7755102040816326, + "fcm_dpo/beta": 0.011425676755607128, + "fcm_dpo/delta": -0.1369817703962326, + "fcm_dpo/margin": 46.925575256347656, + "fcm_dpo/q_t": 0.3841685652732849, + "grad_norm": 13.16512393951416, + "learning_rate": 7.369139731924401e-08, + "logits/chosen": 0.76508629322052, + "logits/rejected": 0.7078437209129333, + "logps/chosen": -129.16046142578125, + "logps/ref_chosen": -50.85256576538086, + "logps/ref_rejected": -69.21754455566406, + "logps/rejected": -194.45101928710938, + "loss": 1.033, + "margin_dpo/margin_mean": 46.925575256347656, + "margin_dpo/margin_std": 62.552696228027344, + "step": 513 + }, + { + "KL/chosen_KL_mean": -87.68719482421875, + "KL/mean": -115.34185028076172, + "KL/rejected_KL_mean": -142.99652099609375, + "KL/std": 63.474998474121094, + "epoch": 0.7770219198790628, + "fcm_dpo/beta": 0.011388057842850685, + "fcm_dpo/delta": -0.23125991225242615, + "fcm_dpo/margin": 55.30931091308594, + "fcm_dpo/q_t": 0.36354586482048035, + "grad_norm": 14.570250511169434, + "learning_rate": 7.275644829568747e-08, + "logits/chosen": 0.6770196557044983, + "logits/rejected": 0.6379963159561157, + "logps/chosen": -157.07212829589844, + "logps/ref_chosen": -69.38493347167969, + "logps/ref_rejected": -83.32447814941406, + "logps/rejected": -226.32098388671875, + "loss": 0.9871, + "margin_dpo/margin_mean": 55.3093147277832, + "margin_dpo/margin_std": 68.06592559814453, + "step": 514 + }, + { + "KL/chosen_KL_mean": -96.31197357177734, + "KL/mean": -119.05532836914062, + "KL/rejected_KL_mean": -141.7987060546875, + "KL/std": 61.81776428222656, + "epoch": 0.7785336356764928, + "fcm_dpo/beta": 0.011346369981765747, + "fcm_dpo/delta": -0.11676906049251556, + "fcm_dpo/margin": 45.486724853515625, + "fcm_dpo/q_t": 0.38847994804382324, + "grad_norm": 15.984508514404297, + "learning_rate": 7.182645715528435e-08, + "logits/chosen": 0.6718851327896118, + "logits/rejected": 0.5861712694168091, + "logps/chosen": -149.99900817871094, + "logps/ref_chosen": -53.687034606933594, + "logps/ref_rejected": -83.59614562988281, + "logps/rejected": -225.39483642578125, + "loss": 1.0687, + "margin_dpo/margin_mean": 45.486724853515625, + "margin_dpo/margin_std": 67.62239074707031, + "step": 515 + }, + { + "KL/chosen_KL_mean": -83.33512878417969, + "KL/mean": -103.34124755859375, + "KL/rejected_KL_mean": -123.34736633300781, + "KL/std": 60.31319808959961, + "epoch": 0.780045351473923, + "fcm_dpo/beta": 0.011338306590914726, + "fcm_dpo/delta": -0.05392259359359741, + "fcm_dpo/margin": 40.012237548828125, + "fcm_dpo/q_t": 0.39862287044525146, + "grad_norm": 17.99725914001465, + "learning_rate": 7.090144991188568e-08, + "logits/chosen": 0.6378867030143738, + "logits/rejected": 0.5942021608352661, + "logps/chosen": -140.23684692382812, + "logps/ref_chosen": -56.9017219543457, + "logps/ref_rejected": -67.83477783203125, + "logps/rejected": -191.18214416503906, + "loss": 1.1262, + "margin_dpo/margin_mean": 40.012237548828125, + "margin_dpo/margin_std": 69.3785400390625, + "step": 516 + }, + { + "KL/chosen_KL_mean": -100.88897705078125, + "KL/mean": -118.00499725341797, + "KL/rejected_KL_mean": -135.12100219726562, + "KL/std": 62.49724578857422, + "epoch": 0.781557067271353, + "fcm_dpo/beta": 0.011337793432176113, + "fcm_dpo/delta": 0.011891733855009079, + "fcm_dpo/margin": 34.23204040527344, + "fcm_dpo/q_t": 0.4171842038631439, + "grad_norm": 17.894546508789062, + "learning_rate": 6.998145243993284e-08, + "logits/chosen": 0.694430410861969, + "logits/rejected": 0.6898482441902161, + "logps/chosen": -162.66412353515625, + "logps/ref_chosen": -61.775142669677734, + "logps/ref_rejected": -62.88270950317383, + "logps/rejected": -198.00372314453125, + "loss": 1.1566, + "margin_dpo/margin_mean": 34.23204040527344, + "margin_dpo/margin_std": 64.19627380371094, + "step": 517 + }, + { + "KL/chosen_KL_mean": -82.73817443847656, + "KL/mean": -103.21205139160156, + "KL/rejected_KL_mean": -123.68592834472656, + "KL/std": 62.375675201416016, + "epoch": 0.783068783068783, + "fcm_dpo/beta": 0.011323518119752407, + "fcm_dpo/delta": -0.06396646052598953, + "fcm_dpo/margin": 40.94774627685547, + "fcm_dpo/q_t": 0.40041807293891907, + "grad_norm": 13.214207649230957, + "learning_rate": 6.906649047373245e-08, + "logits/chosen": 0.6462798118591309, + "logits/rejected": 0.593433141708374, + "logps/chosen": -144.76341247558594, + "logps/ref_chosen": -62.02523422241211, + "logps/ref_rejected": -79.06085205078125, + "logps/rejected": -202.7467803955078, + "loss": 1.1002, + "margin_dpo/margin_mean": 40.947750091552734, + "margin_dpo/margin_std": 65.83390045166016, + "step": 518 + }, + { + "KL/chosen_KL_mean": -101.54141998291016, + "KL/mean": -115.16548156738281, + "KL/rejected_KL_mean": -128.78955078125, + "KL/std": 63.647789001464844, + "epoch": 0.7845804988662132, + "fcm_dpo/beta": 0.01132110133767128, + "fcm_dpo/delta": 0.015766030177474022, + "fcm_dpo/margin": 27.248119354248047, + "fcm_dpo/q_t": 0.43342339992523193, + "grad_norm": 22.26511573791504, + "learning_rate": 6.815658960673781e-08, + "logits/chosen": 0.6644526720046997, + "logits/rejected": 0.6057780981063843, + "logps/chosen": -163.14779663085938, + "logps/ref_chosen": -61.60636901855469, + "logps/ref_rejected": -74.50727844238281, + "logps/rejected": -203.29681396484375, + "loss": 1.2958, + "margin_dpo/margin_mean": 27.248119354248047, + "margin_dpo/margin_std": 81.06497192382812, + "step": 519 + }, + { + "KL/chosen_KL_mean": -93.04032897949219, + "KL/mean": -112.0760498046875, + "KL/rejected_KL_mean": -131.11178588867188, + "KL/std": 63.007110595703125, + "epoch": 0.7860922146636432, + "fcm_dpo/beta": 0.011310771107673645, + "fcm_dpo/delta": -0.03084268420934677, + "fcm_dpo/margin": 38.07143783569336, + "fcm_dpo/q_t": 0.40598154067993164, + "grad_norm": 16.243080139160156, + "learning_rate": 6.725177529083209e-08, + "logits/chosen": 0.7299946546554565, + "logits/rejected": 0.6673400402069092, + "logps/chosen": -155.91375732421875, + "logps/ref_chosen": -62.87343215942383, + "logps/ref_rejected": -76.505615234375, + "logps/rejected": -207.6173858642578, + "loss": 1.1218, + "margin_dpo/margin_mean": 38.07143783569336, + "margin_dpo/margin_std": 63.68070602416992, + "step": 520 + }, + { + "KL/chosen_KL_mean": -91.65220642089844, + "KL/mean": -118.63349151611328, + "KL/rejected_KL_mean": -145.61477661132812, + "KL/std": 61.729652404785156, + "epoch": 0.7876039304610734, + "fcm_dpo/beta": 0.011283602565526962, + "fcm_dpo/delta": -0.2101278007030487, + "fcm_dpo/margin": 53.96255874633789, + "fcm_dpo/q_t": 0.36687666177749634, + "grad_norm": 17.006925582885742, + "learning_rate": 6.63520728356167e-08, + "logits/chosen": 0.5182739496231079, + "logits/rejected": 0.4350222945213318, + "logps/chosen": -155.85888671875, + "logps/ref_chosen": -64.20668029785156, + "logps/ref_rejected": -92.28083038330078, + "logps/rejected": -237.89559936523438, + "loss": 0.9828, + "margin_dpo/margin_mean": 53.962554931640625, + "margin_dpo/margin_std": 63.91650390625, + "step": 521 + }, + { + "KL/chosen_KL_mean": -91.23556518554688, + "KL/mean": -110.56535339355469, + "KL/rejected_KL_mean": -129.8951416015625, + "KL/std": 65.64321899414062, + "epoch": 0.7891156462585034, + "fcm_dpo/beta": 0.011259898543357849, + "fcm_dpo/delta": -0.035475488752126694, + "fcm_dpo/margin": 38.65956115722656, + "fcm_dpo/q_t": 0.4083726406097412, + "grad_norm": 16.47881507873535, + "learning_rate": 6.545750740770336e-08, + "logits/chosen": 0.6281790733337402, + "logits/rejected": 0.621008038520813, + "logps/chosen": -149.60528564453125, + "logps/ref_chosen": -58.369720458984375, + "logps/ref_rejected": -68.79248046875, + "logps/rejected": -198.6876220703125, + "loss": 1.1937, + "margin_dpo/margin_mean": 38.65956115722656, + "margin_dpo/margin_std": 82.96134185791016, + "step": 522 + }, + { + "KL/chosen_KL_mean": -95.62631225585938, + "KL/mean": -117.39295959472656, + "KL/rejected_KL_mean": -139.15960693359375, + "KL/std": 63.40739822387695, + "epoch": 0.7906273620559335, + "fcm_dpo/beta": 0.011241540312767029, + "fcm_dpo/delta": -0.08987335115671158, + "fcm_dpo/margin": 43.53329849243164, + "fcm_dpo/q_t": 0.38979649543762207, + "grad_norm": 17.915002822875977, + "learning_rate": 6.456810403001012e-08, + "logits/chosen": 0.6671806573867798, + "logits/rejected": 0.5339497327804565, + "logps/chosen": -161.33953857421875, + "logps/ref_chosen": -65.71324157714844, + "logps/ref_rejected": -91.98896789550781, + "logps/rejected": -231.14859008789062, + "loss": 1.0988, + "margin_dpo/margin_mean": 43.53329849243164, + "margin_dpo/margin_std": 69.96609497070312, + "step": 523 + }, + { + "KL/chosen_KL_mean": -79.693603515625, + "KL/mean": -101.09695434570312, + "KL/rejected_KL_mean": -122.50028991699219, + "KL/std": 59.759857177734375, + "epoch": 0.7921390778533636, + "fcm_dpo/beta": 0.01123172976076603, + "fcm_dpo/delta": -0.08118852227926254, + "fcm_dpo/margin": 42.80668640136719, + "fcm_dpo/q_t": 0.39249083399772644, + "grad_norm": 14.530516624450684, + "learning_rate": 6.368388758106134e-08, + "logits/chosen": 0.5671854019165039, + "logits/rejected": 0.5428692698478699, + "logps/chosen": -156.04486083984375, + "logps/ref_chosen": -76.35124969482422, + "logps/ref_rejected": -89.96072387695312, + "logps/rejected": -212.46102905273438, + "loss": 1.0688, + "margin_dpo/margin_mean": 42.80668640136719, + "margin_dpo/margin_std": 60.36872863769531, + "step": 524 + }, + { + "KL/chosen_KL_mean": -92.60176086425781, + "KL/mean": -109.48844909667969, + "KL/rejected_KL_mean": -126.37512969970703, + "KL/std": 59.86094665527344, + "epoch": 0.7936507936507936, + "fcm_dpo/beta": 0.011225922033190727, + "fcm_dpo/delta": 0.020941264927387238, + "fcm_dpo/margin": 33.77336120605469, + "fcm_dpo/q_t": 0.4181102216243744, + "grad_norm": 19.154687881469727, + "learning_rate": 6.280488279429185e-08, + "logits/chosen": 0.48071229457855225, + "logits/rejected": 0.4740094244480133, + "logps/chosen": -168.09754943847656, + "logps/ref_chosen": -75.49578857421875, + "logps/ref_rejected": -84.04852294921875, + "logps/rejected": -210.42364501953125, + "loss": 1.1689, + "margin_dpo/margin_mean": 33.77336120605469, + "margin_dpo/margin_std": 66.36637115478516, + "step": 525 + }, + { + "KL/chosen_KL_mean": -98.26417541503906, + "KL/mean": -115.00865173339844, + "KL/rejected_KL_mean": -131.7531280517578, + "KL/std": 61.325157165527344, + "epoch": 0.7951625094482238, + "fcm_dpo/beta": 0.011206869035959244, + "fcm_dpo/delta": -0.08839485794305801, + "fcm_dpo/margin": 33.48894500732422, + "fcm_dpo/q_t": 0.4154875874519348, + "grad_norm": 15.1702880859375, + "learning_rate": 6.193111425735515e-08, + "logits/chosen": 0.6606582403182983, + "logits/rejected": 0.585889458656311, + "logps/chosen": -159.5565948486328, + "logps/ref_chosen": -61.29241943359375, + "logps/ref_rejected": -82.47763061523438, + "logps/rejected": -214.2307586669922, + "loss": 1.1607, + "margin_dpo/margin_mean": 33.48894500732422, + "margin_dpo/margin_std": 60.49855041503906, + "step": 526 + }, + { + "KL/chosen_KL_mean": -103.11119842529297, + "KL/mean": -117.29290771484375, + "KL/rejected_KL_mean": -131.474609375, + "KL/std": 60.230690002441406, + "epoch": 0.7966742252456538, + "fcm_dpo/beta": 0.011205832473933697, + "fcm_dpo/delta": -0.009248212911188602, + "fcm_dpo/margin": 28.363414764404297, + "fcm_dpo/q_t": 0.4309142231941223, + "grad_norm": 17.87224006652832, + "learning_rate": 6.106260641143546e-08, + "logits/chosen": 0.7485909461975098, + "logits/rejected": 0.6612378358840942, + "logps/chosen": -164.58383178710938, + "logps/ref_chosen": -61.472625732421875, + "logps/ref_rejected": -90.52831268310547, + "logps/rejected": -222.0029296875, + "loss": 1.2246, + "margin_dpo/margin_mean": 28.363414764404297, + "margin_dpo/margin_std": 66.57767486572266, + "step": 527 + }, + { + "KL/chosen_KL_mean": -92.66033172607422, + "KL/mean": -107.53262329101562, + "KL/rejected_KL_mean": -122.4049301147461, + "KL/std": 60.551429748535156, + "epoch": 0.7981859410430839, + "fcm_dpo/beta": 0.011211246252059937, + "fcm_dpo/delta": 0.0667407214641571, + "fcm_dpo/margin": 29.744598388671875, + "fcm_dpo/q_t": 0.4252808392047882, + "grad_norm": 18.81622886657715, + "learning_rate": 6.019938355056422e-08, + "logits/chosen": 0.5815372467041016, + "logits/rejected": 0.4922967553138733, + "logps/chosen": -151.4523468017578, + "logps/ref_chosen": -58.792015075683594, + "logps/ref_rejected": -71.82516479492188, + "logps/rejected": -194.2301025390625, + "loss": 1.2347, + "margin_dpo/margin_mean": 29.74459457397461, + "margin_dpo/margin_std": 72.06694030761719, + "step": 528 + }, + { + "KL/chosen_KL_mean": -88.04642486572266, + "KL/mean": -121.09292602539062, + "KL/rejected_KL_mean": -154.13943481445312, + "KL/std": 62.96925354003906, + "epoch": 0.799697656840514, + "fcm_dpo/beta": 0.011159720830619335, + "fcm_dpo/delta": -0.3401945233345032, + "fcm_dpo/margin": 66.09300994873047, + "fcm_dpo/q_t": 0.3409721851348877, + "grad_norm": 13.887116432189941, + "learning_rate": 5.934146982094049e-08, + "logits/chosen": 0.5674487352371216, + "logits/rejected": 0.5093246698379517, + "logps/chosen": -143.1173858642578, + "logps/ref_chosen": -55.070960998535156, + "logps/ref_rejected": -75.44007873535156, + "logps/rejected": -229.57949829101562, + "loss": 0.9132, + "margin_dpo/margin_mean": 66.09300231933594, + "margin_dpo/margin_std": 69.58425903320312, + "step": 529 + }, + { + "KL/chosen_KL_mean": -89.5904541015625, + "KL/mean": -109.80479431152344, + "KL/rejected_KL_mean": -130.01914978027344, + "KL/std": 63.00408935546875, + "epoch": 0.8012093726379441, + "fcm_dpo/beta": 0.011137645691633224, + "fcm_dpo/delta": -0.05050615966320038, + "fcm_dpo/margin": 40.42868423461914, + "fcm_dpo/q_t": 0.4022928774356842, + "grad_norm": 17.666433334350586, + "learning_rate": 5.848888922025552e-08, + "logits/chosen": 0.6687257289886475, + "logits/rejected": 0.6220978498458862, + "logps/chosen": -146.33425903320312, + "logps/ref_chosen": -56.743812561035156, + "logps/ref_rejected": -76.6692123413086, + "logps/rejected": -206.6883544921875, + "loss": 1.0968, + "margin_dpo/margin_mean": 40.42868423461914, + "margin_dpo/margin_std": 61.98796081542969, + "step": 530 + }, + { + "KL/chosen_KL_mean": -90.4144287109375, + "KL/mean": -111.25164031982422, + "KL/rejected_KL_mean": -132.08883666992188, + "KL/std": 64.20317077636719, + "epoch": 0.8027210884353742, + "fcm_dpo/beta": 0.011129481717944145, + "fcm_dpo/delta": -0.06416445970535278, + "fcm_dpo/margin": 41.67441177368164, + "fcm_dpo/q_t": 0.3993687629699707, + "grad_norm": 16.592185974121094, + "learning_rate": 5.7641665597021435e-08, + "logits/chosen": 0.6512601375579834, + "logits/rejected": 0.5735520124435425, + "logps/chosen": -141.5308837890625, + "logps/ref_chosen": -51.116455078125, + "logps/ref_rejected": -79.52884674072266, + "logps/rejected": -211.61769104003906, + "loss": 1.1045, + "margin_dpo/margin_mean": 41.67441177368164, + "margin_dpo/margin_std": 66.7490234375, + "step": 531 + }, + { + "KL/chosen_KL_mean": -105.46018981933594, + "KL/mean": -128.8103790283203, + "KL/rejected_KL_mean": -152.16058349609375, + "KL/std": 61.70550537109375, + "epoch": 0.8042328042328042, + "fcm_dpo/beta": 0.011097250506281853, + "fcm_dpo/delta": -0.11887596547603607, + "fcm_dpo/margin": 46.70039367675781, + "fcm_dpo/q_t": 0.38727831840515137, + "grad_norm": 14.97189712524414, + "learning_rate": 5.679982264990424e-08, + "logits/chosen": 0.5733745098114014, + "logits/rejected": 0.5172688961029053, + "logps/chosen": -163.74014282226562, + "logps/ref_chosen": -58.279945373535156, + "logps/ref_rejected": -78.05426788330078, + "logps/rejected": -230.21484375, + "loss": 1.0741, + "margin_dpo/margin_mean": 46.70039367675781, + "margin_dpo/margin_std": 70.89237976074219, + "step": 532 + }, + { + "KL/chosen_KL_mean": -73.63652038574219, + "KL/mean": -95.60624694824219, + "KL/rejected_KL_mean": -117.57598114013672, + "KL/std": 63.09911346435547, + "epoch": 0.8057445200302343, + "fcm_dpo/beta": 0.011079625226557255, + "fcm_dpo/delta": -0.08724816143512726, + "fcm_dpo/margin": 43.93946838378906, + "fcm_dpo/q_t": 0.39221304655075073, + "grad_norm": 13.915875434875488, + "learning_rate": 5.596338392706076e-08, + "logits/chosen": 0.7704240083694458, + "logits/rejected": 0.6960223317146301, + "logps/chosen": -130.05453491210938, + "logps/ref_chosen": -56.41801071166992, + "logps/ref_rejected": -73.89324951171875, + "logps/rejected": -191.46923828125, + "loss": 1.0789, + "margin_dpo/margin_mean": 43.93946838378906, + "margin_dpo/margin_std": 65.2164306640625, + "step": 533 + }, + { + "KL/chosen_KL_mean": -87.94291687011719, + "KL/mean": -111.34124755859375, + "KL/rejected_KL_mean": -134.7395782470703, + "KL/std": 63.306304931640625, + "epoch": 0.8072562358276644, + "fcm_dpo/beta": 0.011048360727727413, + "fcm_dpo/delta": -0.11771686375141144, + "fcm_dpo/margin": 46.796669006347656, + "fcm_dpo/q_t": 0.38955187797546387, + "grad_norm": 13.438860893249512, + "learning_rate": 5.513237282548033e-08, + "logits/chosen": 0.650261402130127, + "logits/rejected": 0.6106135845184326, + "logps/chosen": -148.6916046142578, + "logps/ref_chosen": -60.748687744140625, + "logps/ref_rejected": -73.8623046875, + "logps/rejected": -208.6018829345703, + "loss": 1.0828, + "margin_dpo/margin_mean": 46.79667663574219, + "margin_dpo/margin_std": 73.54011535644531, + "step": 534 + }, + { + "KL/chosen_KL_mean": -99.77320861816406, + "KL/mean": -118.77926635742188, + "KL/rejected_KL_mean": -137.7853240966797, + "KL/std": 66.09870910644531, + "epoch": 0.8087679516250945, + "fcm_dpo/beta": 0.011045056395232677, + "fcm_dpo/delta": -0.019932501018047333, + "fcm_dpo/margin": 38.012115478515625, + "fcm_dpo/q_t": 0.40844523906707764, + "grad_norm": 16.677663803100586, + "learning_rate": 5.430681259032957e-08, + "logits/chosen": 0.5195807218551636, + "logits/rejected": 0.4538642466068268, + "logps/chosen": -161.41061401367188, + "logps/ref_chosen": -61.637413024902344, + "logps/ref_rejected": -80.93138885498047, + "logps/rejected": -218.7167205810547, + "loss": 1.1492, + "margin_dpo/margin_mean": 38.012115478515625, + "margin_dpo/margin_std": 70.58258056640625, + "step": 535 + }, + { + "KL/chosen_KL_mean": -88.17255401611328, + "KL/mean": -116.95718383789062, + "KL/rejected_KL_mean": -145.74179077148438, + "KL/std": 66.51947021484375, + "epoch": 0.8102796674225246, + "fcm_dpo/beta": 0.011002978309988976, + "fcm_dpo/delta": -0.23492556810379028, + "fcm_dpo/margin": 57.569236755371094, + "fcm_dpo/q_t": 0.3617976903915405, + "grad_norm": 11.167461395263672, + "learning_rate": 5.3486726314303175e-08, + "logits/chosen": 0.6817443370819092, + "logits/rejected": 0.5870887637138367, + "logps/chosen": -140.0615234375, + "logps/ref_chosen": -51.88897705078125, + "logps/ref_rejected": -73.34864044189453, + "logps/rejected": -219.09043884277344, + "loss": 0.947, + "margin_dpo/margin_mean": 57.569236755371094, + "margin_dpo/margin_std": 60.594383239746094, + "step": 536 + }, + { + "KL/chosen_KL_mean": -98.80126190185547, + "KL/mean": -121.23102569580078, + "KL/rejected_KL_mean": -143.66079711914062, + "KL/std": 68.20759582519531, + "epoch": 0.8117913832199547, + "fcm_dpo/beta": 0.010970618575811386, + "fcm_dpo/delta": -0.09266072511672974, + "fcm_dpo/margin": 44.85953903198242, + "fcm_dpo/q_t": 0.39473938941955566, + "grad_norm": 13.600377082824707, + "learning_rate": 5.267213693697695e-08, + "logits/chosen": 0.7365930080413818, + "logits/rejected": 0.6363020539283752, + "logps/chosen": -153.0498809814453, + "logps/ref_chosen": -54.248619079589844, + "logps/ref_rejected": -94.94343566894531, + "logps/rejected": -238.60423278808594, + "loss": 1.0861, + "margin_dpo/margin_mean": 44.85953903198242, + "margin_dpo/margin_std": 69.64356994628906, + "step": 537 + }, + { + "KL/chosen_KL_mean": -95.9029769897461, + "KL/mean": -121.42750549316406, + "KL/rejected_KL_mean": -146.95204162597656, + "KL/std": 65.03889465332031, + "epoch": 0.8133030990173847, + "fcm_dpo/beta": 0.01094746869057417, + "fcm_dpo/delta": -0.1597195267677307, + "fcm_dpo/margin": 51.04906463623047, + "fcm_dpo/q_t": 0.37766122817993164, + "grad_norm": 14.633785247802734, + "learning_rate": 5.1863067244167144e-08, + "logits/chosen": 0.6539440155029297, + "logits/rejected": 0.6211650967597961, + "logps/chosen": -165.99652099609375, + "logps/ref_chosen": -70.09353637695312, + "logps/ref_rejected": -79.49833679199219, + "logps/rejected": -226.45037841796875, + "loss": 1.0195, + "margin_dpo/margin_mean": 51.04906463623047, + "margin_dpo/margin_std": 65.70713806152344, + "step": 538 + }, + { + "KL/chosen_KL_mean": -100.50103759765625, + "KL/mean": -119.245849609375, + "KL/rejected_KL_mean": -137.9906768798828, + "KL/std": 63.370296478271484, + "epoch": 0.8148148148148148, + "fcm_dpo/beta": 0.010929939337074757, + "fcm_dpo/delta": -0.009821347892284393, + "fcm_dpo/margin": 37.489646911621094, + "fcm_dpo/q_t": 0.4099830985069275, + "grad_norm": 14.08874797821045, + "learning_rate": 5.105953986729195e-08, + "logits/chosen": 0.5902529954910278, + "logits/rejected": 0.5036922097206116, + "logps/chosen": -162.4327392578125, + "logps/ref_chosen": -61.93169403076172, + "logps/ref_rejected": -84.08946228027344, + "logps/rejected": -222.08013916015625, + "loss": 1.1206, + "margin_dpo/margin_mean": 37.489646911621094, + "margin_dpo/margin_std": 61.40064239501953, + "step": 539 + }, + { + "KL/chosen_KL_mean": -92.65571594238281, + "KL/mean": -120.20235443115234, + "KL/rejected_KL_mean": -147.74899291992188, + "KL/std": 70.48026275634766, + "epoch": 0.8163265306122449, + "fcm_dpo/beta": 0.010907829739153385, + "fcm_dpo/delta": -0.20212388038635254, + "fcm_dpo/margin": 55.09327697753906, + "fcm_dpo/q_t": 0.3690637946128845, + "grad_norm": 13.944159507751465, + "learning_rate": 5.026157728273966e-08, + "logits/chosen": 0.6962717771530151, + "logits/rejected": 0.5875642895698547, + "logps/chosen": -155.35997009277344, + "logps/ref_chosen": -62.704254150390625, + "logps/ref_rejected": -95.63597106933594, + "logps/rejected": -243.3849639892578, + "loss": 0.9849, + "margin_dpo/margin_mean": 55.09327697753906, + "margin_dpo/margin_std": 63.60958480834961, + "step": 540 + }, + { + "KL/chosen_KL_mean": -88.39573669433594, + "KL/mean": -115.96863555908203, + "KL/rejected_KL_mean": -143.54153442382812, + "KL/std": 66.36589050292969, + "epoch": 0.817838246409675, + "fcm_dpo/beta": 0.010849235579371452, + "fcm_dpo/delta": -0.19968968629837036, + "fcm_dpo/margin": 55.14580535888672, + "fcm_dpo/q_t": 0.36822330951690674, + "grad_norm": 14.913622856140137, + "learning_rate": 4.9469201811239035e-08, + "logits/chosen": 0.6747847199440002, + "logits/rejected": 0.6968380808830261, + "logps/chosen": -150.8765869140625, + "logps/ref_chosen": -62.48084259033203, + "logps/ref_rejected": -57.55541229248047, + "logps/rejected": -201.09695434570312, + "loss": 0.9805, + "margin_dpo/margin_mean": 55.14580535888672, + "margin_dpo/margin_std": 61.894874572753906, + "step": 541 + }, + { + "KL/chosen_KL_mean": -77.14692687988281, + "KL/mean": -106.20926666259766, + "KL/rejected_KL_mean": -135.2716064453125, + "KL/std": 64.15353393554688, + "epoch": 0.8193499622071051, + "fcm_dpo/beta": 0.010807383805513382, + "fcm_dpo/delta": -0.22963140904903412, + "fcm_dpo/margin": 58.124671936035156, + "fcm_dpo/q_t": 0.3650358319282532, + "grad_norm": 13.22535228729248, + "learning_rate": 4.868243561723534e-08, + "logits/chosen": 0.7041116952896118, + "logits/rejected": 0.6500225067138672, + "logps/chosen": -126.60182189941406, + "logps/ref_chosen": -49.454891204833984, + "logps/ref_rejected": -65.33275604248047, + "logps/rejected": -200.6043701171875, + "loss": 0.9988, + "margin_dpo/margin_mean": 58.12467575073242, + "margin_dpo/margin_std": 74.76990509033203, + "step": 542 + }, + { + "KL/chosen_KL_mean": -85.88755798339844, + "KL/mean": -111.61042022705078, + "KL/rejected_KL_mean": -137.33328247070312, + "KL/std": 62.4495849609375, + "epoch": 0.8208616780045351, + "fcm_dpo/beta": 0.01077445037662983, + "fcm_dpo/delta": -0.15512952208518982, + "fcm_dpo/margin": 51.44572448730469, + "fcm_dpo/q_t": 0.3759421706199646, + "grad_norm": 11.267179489135742, + "learning_rate": 4.790130070827028e-08, + "logits/chosen": 0.6515238285064697, + "logits/rejected": 0.5555661916732788, + "logps/chosen": -136.98841857910156, + "logps/ref_chosen": -51.100860595703125, + "logps/ref_rejected": -76.06130981445312, + "logps/rejected": -213.39459228515625, + "loss": 1.0221, + "margin_dpo/margin_mean": 51.44572448730469, + "margin_dpo/margin_std": 65.00933837890625, + "step": 543 + }, + { + "KL/chosen_KL_mean": -93.42378234863281, + "KL/mean": -123.00735473632812, + "KL/rejected_KL_mean": -152.5909423828125, + "KL/std": 67.50820922851562, + "epoch": 0.8223733938019653, + "fcm_dpo/beta": 0.010724778287112713, + "fcm_dpo/delta": -0.2360384166240692, + "fcm_dpo/margin": 59.167152404785156, + "fcm_dpo/q_t": 0.3635045886039734, + "grad_norm": 13.828411102294922, + "learning_rate": 4.7125818934366454e-08, + "logits/chosen": 0.6397875547409058, + "logits/rejected": 0.5591890215873718, + "logps/chosen": -153.70101928710938, + "logps/ref_chosen": -60.2772331237793, + "logps/ref_rejected": -88.40553283691406, + "logps/rejected": -240.9964599609375, + "loss": 0.9855, + "margin_dpo/margin_mean": 59.16715621948242, + "margin_dpo/margin_std": 73.09345245361328, + "step": 544 + }, + { + "KL/chosen_KL_mean": -98.35765075683594, + "KL/mean": -116.91368103027344, + "KL/rejected_KL_mean": -135.46969604492188, + "KL/std": 65.88688659667969, + "epoch": 0.8238851095993953, + "fcm_dpo/beta": 0.01071164570748806, + "fcm_dpo/delta": 0.002478633774444461, + "fcm_dpo/margin": 37.112060546875, + "fcm_dpo/q_t": 0.41300711035728455, + "grad_norm": 14.4955472946167, + "learning_rate": 4.635601198741607e-08, + "logits/chosen": 0.6188709735870361, + "logits/rejected": 0.5560128092765808, + "logps/chosen": -159.972900390625, + "logps/ref_chosen": -61.61524963378906, + "logps/ref_rejected": -78.71266174316406, + "logps/rejected": -214.182373046875, + "loss": 1.1561, + "margin_dpo/margin_mean": 37.112060546875, + "margin_dpo/margin_std": 69.28641510009766, + "step": 545 + }, + { + "KL/chosen_KL_mean": -91.83230590820312, + "KL/mean": -115.2255859375, + "KL/rejected_KL_mean": -138.618896484375, + "KL/std": 60.86692810058594, + "epoch": 0.8253968253968254, + "fcm_dpo/beta": 0.010694079101085663, + "fcm_dpo/delta": -0.10086473822593689, + "fcm_dpo/margin": 46.786582946777344, + "fcm_dpo/q_t": 0.38844865560531616, + "grad_norm": 16.732141494750977, + "learning_rate": 4.559190140057428e-08, + "logits/chosen": 0.7269467711448669, + "logits/rejected": 0.7185274958610535, + "logps/chosen": -151.14556884765625, + "logps/ref_chosen": -59.313262939453125, + "logps/ref_rejected": -64.73631286621094, + "logps/rejected": -203.35519409179688, + "loss": 1.0675, + "margin_dpo/margin_mean": 46.786582946777344, + "margin_dpo/margin_std": 67.49736022949219, + "step": 546 + }, + { + "KL/chosen_KL_mean": -83.7637939453125, + "KL/mean": -111.42105102539062, + "KL/rejected_KL_mean": -139.07830810546875, + "KL/std": 64.22439575195312, + "epoch": 0.8269085411942555, + "fcm_dpo/beta": 0.010658136568963528, + "fcm_dpo/delta": -0.19070938229560852, + "fcm_dpo/margin": 55.31450271606445, + "fcm_dpo/q_t": 0.36983001232147217, + "grad_norm": 16.78713035583496, + "learning_rate": 4.483350854765672e-08, + "logits/chosen": 0.5930368900299072, + "logits/rejected": 0.5223778486251831, + "logps/chosen": -138.74053955078125, + "logps/ref_chosen": -54.97674560546875, + "logps/ref_rejected": -75.35922241210938, + "logps/rejected": -214.43753051757812, + "loss": 1.0076, + "margin_dpo/margin_mean": 55.31450653076172, + "margin_dpo/margin_std": 69.74282836914062, + "step": 547 + }, + { + "KL/chosen_KL_mean": -97.19854736328125, + "KL/mean": -115.417236328125, + "KL/rejected_KL_mean": -133.63592529296875, + "KL/std": 64.96145629882812, + "epoch": 0.8284202569916855, + "fcm_dpo/beta": 0.010656967759132385, + "fcm_dpo/delta": 0.011675331741571426, + "fcm_dpo/margin": 36.43736267089844, + "fcm_dpo/q_t": 0.4161408543586731, + "grad_norm": 15.998262405395508, + "learning_rate": 4.4080854642541826e-08, + "logits/chosen": 0.5870075821876526, + "logits/rejected": 0.5149146914482117, + "logps/chosen": -160.4092254638672, + "logps/ref_chosen": -63.21067428588867, + "logps/ref_rejected": -81.23347473144531, + "logps/rejected": -214.869384765625, + "loss": 1.16, + "margin_dpo/margin_mean": 36.43735885620117, + "margin_dpo/margin_std": 68.64096069335938, + "step": 548 + }, + { + "KL/chosen_KL_mean": -95.11773681640625, + "KL/mean": -114.94932556152344, + "KL/rejected_KL_mean": -134.78091430664062, + "KL/std": 64.93453979492188, + "epoch": 0.8299319727891157, + "fcm_dpo/beta": 0.010646875947713852, + "fcm_dpo/delta": -0.022392230108380318, + "fcm_dpo/margin": 39.6631965637207, + "fcm_dpo/q_t": 0.4059330224990845, + "grad_norm": 15.384112358093262, + "learning_rate": 4.333396073857723e-08, + "logits/chosen": 0.6852062940597534, + "logits/rejected": 0.6105310916900635, + "logps/chosen": -159.39125061035156, + "logps/ref_chosen": -64.27351379394531, + "logps/ref_rejected": -92.31663513183594, + "logps/rejected": -227.09756469726562, + "loss": 1.1374, + "margin_dpo/margin_mean": 39.66319274902344, + "margin_dpo/margin_std": 70.17346954345703, + "step": 549 + }, + { + "KL/chosen_KL_mean": -103.2861099243164, + "KL/mean": -118.31254577636719, + "KL/rejected_KL_mean": -133.33895874023438, + "KL/std": 63.28034210205078, + "epoch": 0.8314436885865457, + "fcm_dpo/beta": 0.01064311619848013, + "fcm_dpo/delta": -0.019979851320385933, + "fcm_dpo/margin": 30.052852630615234, + "fcm_dpo/q_t": 0.4282895624637604, + "grad_norm": 15.491739273071289, + "learning_rate": 4.259284772799099e-08, + "logits/chosen": 0.6756773591041565, + "logits/rejected": 0.6422036290168762, + "logps/chosen": -159.51654052734375, + "logps/ref_chosen": -56.230438232421875, + "logps/ref_rejected": -62.59788513183594, + "logps/rejected": -195.93685913085938, + "loss": 1.2025, + "margin_dpo/margin_mean": 30.052852630615234, + "margin_dpo/margin_std": 63.91701126098633, + "step": 550 + }, + { + "KL/chosen_KL_mean": -99.15203857421875, + "KL/mean": -118.67096710205078, + "KL/rejected_KL_mean": -138.18988037109375, + "KL/std": 65.1424789428711, + "epoch": 0.8329554043839759, + "fcm_dpo/beta": 0.010643774643540382, + "fcm_dpo/delta": -0.015587572008371353, + "fcm_dpo/margin": 39.037837982177734, + "fcm_dpo/q_t": 0.4095955491065979, + "grad_norm": 14.788942337036133, + "learning_rate": 4.1857536341307176e-08, + "logits/chosen": 0.6847094297409058, + "logits/rejected": 0.6470929384231567, + "logps/chosen": -166.89926147460938, + "logps/ref_chosen": -67.74720764160156, + "logps/ref_rejected": -87.04285430908203, + "logps/rejected": -225.2327423095703, + "loss": 1.1102, + "margin_dpo/margin_mean": 39.037841796875, + "margin_dpo/margin_std": 62.46849060058594, + "step": 551 + }, + { + "KL/chosen_KL_mean": -92.89454650878906, + "KL/mean": -116.33419799804688, + "KL/rejected_KL_mean": -139.77383422851562, + "KL/std": 65.33940124511719, + "epoch": 0.8344671201814059, + "fcm_dpo/beta": 0.010628802701830864, + "fcm_dpo/delta": -0.09875884652137756, + "fcm_dpo/margin": 46.87930679321289, + "fcm_dpo/q_t": 0.3851057291030884, + "grad_norm": 16.932785034179688, + "learning_rate": 4.112804714676593e-08, + "logits/chosen": 0.6310358047485352, + "logits/rejected": 0.5728182792663574, + "logps/chosen": -155.82080078125, + "logps/ref_chosen": -62.92625427246094, + "logps/ref_rejected": -82.98365783691406, + "logps/rejected": -222.75750732421875, + "loss": 1.0493, + "margin_dpo/margin_mean": 46.879302978515625, + "margin_dpo/margin_std": 61.433074951171875, + "step": 552 + }, + { + "KL/chosen_KL_mean": -101.81392669677734, + "KL/mean": -122.57499694824219, + "KL/rejected_KL_mean": -143.3360595703125, + "KL/std": 64.75807189941406, + "epoch": 0.8359788359788359, + "fcm_dpo/beta": 0.010611262172460556, + "fcm_dpo/delta": -0.04078802838921547, + "fcm_dpo/margin": 41.52214050292969, + "fcm_dpo/q_t": 0.40576934814453125, + "grad_norm": 17.67205238342285, + "learning_rate": 4.0404400549748144e-08, + "logits/chosen": 0.6215210556983948, + "logits/rejected": 0.5048577189445496, + "logps/chosen": -157.8524169921875, + "logps/ref_chosen": -56.038490295410156, + "logps/ref_rejected": -84.48454284667969, + "logps/rejected": -227.8206024169922, + "loss": 1.1368, + "margin_dpo/margin_mean": 41.52214050292969, + "margin_dpo/margin_std": 75.44398498535156, + "step": 553 + }, + { + "KL/chosen_KL_mean": -92.83740234375, + "KL/mean": -118.1080322265625, + "KL/rejected_KL_mean": -143.37867736816406, + "KL/std": 62.669677734375, + "epoch": 0.8374905517762661, + "fcm_dpo/beta": 0.010588699951767921, + "fcm_dpo/delta": -0.13589191436767578, + "fcm_dpo/margin": 50.54127502441406, + "fcm_dpo/q_t": 0.3821818232536316, + "grad_norm": 14.348846435546875, + "learning_rate": 3.968661679220467e-08, + "logits/chosen": 0.6118894815444946, + "logits/rejected": 0.5894501209259033, + "logps/chosen": -157.3679962158203, + "logps/ref_chosen": -64.53059387207031, + "logps/ref_rejected": -71.2155990600586, + "logps/rejected": -214.59426879882812, + "loss": 1.0449, + "margin_dpo/margin_mean": 50.54127502441406, + "margin_dpo/margin_std": 69.63795471191406, + "step": 554 + }, + { + "KL/chosen_KL_mean": -102.38626098632812, + "KL/mean": -124.24322509765625, + "KL/rejected_KL_mean": -146.10018920898438, + "KL/std": 64.1204833984375, + "epoch": 0.8390022675736961, + "fcm_dpo/beta": 0.010581552982330322, + "fcm_dpo/delta": -0.06297705322504044, + "fcm_dpo/margin": 43.71392059326172, + "fcm_dpo/q_t": 0.3959122598171234, + "grad_norm": 15.550765037536621, + "learning_rate": 3.89747159520904e-08, + "logits/chosen": 0.6407837271690369, + "logits/rejected": 0.6107930541038513, + "logps/chosen": -169.03817749023438, + "logps/ref_chosen": -66.65191650390625, + "logps/ref_rejected": -68.6667251586914, + "logps/rejected": -214.76690673828125, + "loss": 1.1303, + "margin_dpo/margin_mean": 43.71392059326172, + "margin_dpo/margin_std": 74.43399047851562, + "step": 555 + }, + { + "KL/chosen_KL_mean": -103.5107421875, + "KL/mean": -122.67743682861328, + "KL/rejected_KL_mean": -141.8441162109375, + "KL/std": 67.56221008300781, + "epoch": 0.8405139833711263, + "fcm_dpo/beta": 0.010564668104052544, + "fcm_dpo/delta": -0.005012618377804756, + "fcm_dpo/margin": 38.333370208740234, + "fcm_dpo/q_t": 0.41347646713256836, + "grad_norm": 13.732193946838379, + "learning_rate": 3.826871794280192e-08, + "logits/chosen": 0.6973532438278198, + "logits/rejected": 0.6397409439086914, + "logps/chosen": -156.34310913085938, + "logps/ref_chosen": -52.832366943359375, + "logps/ref_rejected": -64.49044036865234, + "logps/rejected": -206.33456420898438, + "loss": 1.1661, + "margin_dpo/margin_mean": 38.3333740234375, + "margin_dpo/margin_std": 74.75647735595703, + "step": 556 + }, + { + "KL/chosen_KL_mean": -101.59855651855469, + "KL/mean": -129.11581420898438, + "KL/rejected_KL_mean": -156.63304138183594, + "KL/std": 65.07809448242188, + "epoch": 0.8420256991685563, + "fcm_dpo/beta": 0.01053388137370348, + "fcm_dpo/delta": -0.1808881312608719, + "fcm_dpo/margin": 55.03450012207031, + "fcm_dpo/q_t": 0.3726317286491394, + "grad_norm": 14.326994895935059, + "learning_rate": 3.756864251262143e-08, + "logits/chosen": 0.7543884515762329, + "logits/rejected": 0.669520378112793, + "logps/chosen": -156.63455200195312, + "logps/ref_chosen": -55.03598403930664, + "logps/ref_rejected": -75.80644989013672, + "logps/rejected": -232.43948364257812, + "loss": 0.9882, + "margin_dpo/margin_mean": 55.03449249267578, + "margin_dpo/margin_std": 62.75926971435547, + "step": 557 + }, + { + "KL/chosen_KL_mean": -88.7650146484375, + "KL/mean": -119.16663360595703, + "KL/rejected_KL_mean": -149.5682373046875, + "KL/std": 69.15695190429688, + "epoch": 0.8435374149659864, + "fcm_dpo/beta": 0.010487494990229607, + "fcm_dpo/delta": -0.23930586874485016, + "fcm_dpo/margin": 60.80324172973633, + "fcm_dpo/q_t": 0.36224135756492615, + "grad_norm": 11.825664520263672, + "learning_rate": 3.687450924416341e-08, + "logits/chosen": 0.6914137601852417, + "logits/rejected": 0.632148265838623, + "logps/chosen": -151.99136352539062, + "logps/ref_chosen": -63.226348876953125, + "logps/ref_rejected": -91.46881866455078, + "logps/rejected": -241.0370635986328, + "loss": 0.9602, + "margin_dpo/margin_mean": 60.80324172973633, + "margin_dpo/margin_std": 68.02108764648438, + "step": 558 + }, + { + "KL/chosen_KL_mean": -96.18344116210938, + "KL/mean": -122.18670654296875, + "KL/rejected_KL_mean": -148.18997192382812, + "KL/std": 67.02165985107422, + "epoch": 0.8450491307634165, + "fcm_dpo/beta": 0.010446674190461636, + "fcm_dpo/delta": -0.14436130225658417, + "fcm_dpo/margin": 52.00653076171875, + "fcm_dpo/q_t": 0.385856032371521, + "grad_norm": 12.168778419494629, + "learning_rate": 3.6186337553827743e-08, + "logits/chosen": 0.6509109735488892, + "logits/rejected": 0.5751421451568604, + "logps/chosen": -157.705078125, + "logps/ref_chosen": -61.521644592285156, + "logps/ref_rejected": -82.83859252929688, + "logps/rejected": -231.028564453125, + "loss": 1.0601, + "margin_dpo/margin_mean": 52.00653076171875, + "margin_dpo/margin_std": 76.0902099609375, + "step": 559 + }, + { + "KL/chosen_KL_mean": -102.17713928222656, + "KL/mean": -126.3495864868164, + "KL/rejected_KL_mean": -150.52203369140625, + "KL/std": 65.51622009277344, + "epoch": 0.8465608465608465, + "fcm_dpo/beta": 0.010438122786581516, + "fcm_dpo/delta": -0.1051693707704544, + "fcm_dpo/margin": 48.34488296508789, + "fcm_dpo/q_t": 0.3894042372703552, + "grad_norm": 14.337431907653809, + "learning_rate": 3.550414669125573e-08, + "logits/chosen": 0.6619716286659241, + "logits/rejected": 0.6186896562576294, + "logps/chosen": -162.818359375, + "logps/ref_chosen": -60.64122009277344, + "logps/ref_rejected": -78.75474548339844, + "logps/rejected": -229.2767791748047, + "loss": 1.0527, + "margin_dpo/margin_mean": 48.344886779785156, + "margin_dpo/margin_std": 65.04598999023438, + "step": 560 + }, + { + "KL/chosen_KL_mean": -91.07806396484375, + "KL/mean": -115.9728012084961, + "KL/rejected_KL_mean": -140.86753845214844, + "KL/std": 66.17451477050781, + "epoch": 0.8480725623582767, + "fcm_dpo/beta": 0.01040741428732872, + "fcm_dpo/delta": -0.11880025267601013, + "fcm_dpo/margin": 49.78947448730469, + "fcm_dpo/q_t": 0.38874882459640503, + "grad_norm": 12.99413776397705, + "learning_rate": 3.482795573879241e-08, + "logits/chosen": 0.6080992221832275, + "logits/rejected": 0.5736936330795288, + "logps/chosen": -153.57666015625, + "logps/ref_chosen": -62.49859619140625, + "logps/ref_rejected": -78.72064208984375, + "logps/rejected": -219.5881805419922, + "loss": 1.0546, + "margin_dpo/margin_mean": 49.78947448730469, + "margin_dpo/margin_std": 70.19805908203125, + "step": 561 + }, + { + "KL/chosen_KL_mean": -98.73811340332031, + "KL/mean": -127.15272521972656, + "KL/rejected_KL_mean": -155.5673065185547, + "KL/std": 70.07037353515625, + "epoch": 0.8495842781557067, + "fcm_dpo/beta": 0.010366078466176987, + "fcm_dpo/delta": -0.19039033353328705, + "fcm_dpo/margin": 56.82918930053711, + "fcm_dpo/q_t": 0.37304115295410156, + "grad_norm": 16.6341552734375, + "learning_rate": 3.415778361095226e-08, + "logits/chosen": 0.6189653277397156, + "logits/rejected": 0.5749369859695435, + "logps/chosen": -173.5198516845703, + "logps/ref_chosen": -74.78173828125, + "logps/ref_rejected": -92.63499450683594, + "logps/rejected": -248.20230102539062, + "loss": 1.0037, + "margin_dpo/margin_mean": 56.829193115234375, + "margin_dpo/margin_std": 70.57183074951172, + "step": 562 + }, + { + "KL/chosen_KL_mean": -85.60863494873047, + "KL/mean": -110.69312286376953, + "KL/rejected_KL_mean": -135.77760314941406, + "KL/std": 61.61790466308594, + "epoch": 0.8510959939531368, + "fcm_dpo/beta": 0.010344208218157291, + "fcm_dpo/delta": -0.11957196891307831, + "fcm_dpo/margin": 50.168968200683594, + "fcm_dpo/q_t": 0.38610565662384033, + "grad_norm": 17.92778205871582, + "learning_rate": 3.349364905389032e-08, + "logits/chosen": 0.7401456236839294, + "logits/rejected": 0.6854729652404785, + "logps/chosen": -135.80712890625, + "logps/ref_chosen": -50.19850158691406, + "logps/ref_rejected": -66.76687622070312, + "logps/rejected": -202.5444793701172, + "loss": 1.0682, + "margin_dpo/margin_mean": 50.168968200683594, + "margin_dpo/margin_std": 74.25452423095703, + "step": 563 + }, + { + "KL/chosen_KL_mean": -84.7935791015625, + "KL/mean": -116.21170043945312, + "KL/rejected_KL_mean": -147.62982177734375, + "KL/std": 65.67542266845703, + "epoch": 0.8526077097505669, + "fcm_dpo/beta": 0.010298274457454681, + "fcm_dpo/delta": -0.24872998893260956, + "fcm_dpo/margin": 62.83623123168945, + "fcm_dpo/q_t": 0.35921698808670044, + "grad_norm": 12.321980476379395, + "learning_rate": 3.283557064487785e-08, + "logits/chosen": 0.5830624103546143, + "logits/rejected": 0.549677848815918, + "logps/chosen": -140.534423828125, + "logps/ref_chosen": -55.7408447265625, + "logps/ref_rejected": -74.82323455810547, + "logps/rejected": -222.45306396484375, + "loss": 0.9761, + "margin_dpo/margin_mean": 62.83622741699219, + "margin_dpo/margin_std": 75.44405364990234, + "step": 564 + }, + { + "KL/chosen_KL_mean": -106.71977233886719, + "KL/mean": -127.81595611572266, + "KL/rejected_KL_mean": -148.91213989257812, + "KL/std": 67.79901123046875, + "epoch": 0.854119425547997, + "fcm_dpo/beta": 0.010286583565175533, + "fcm_dpo/delta": -0.03420817479491234, + "fcm_dpo/margin": 42.19237518310547, + "fcm_dpo/q_t": 0.40412914752960205, + "grad_norm": 14.25365924835205, + "learning_rate": 3.218356679178252e-08, + "logits/chosen": 0.6795693039894104, + "logits/rejected": 0.6208063364028931, + "logps/chosen": -165.05715942382812, + "logps/ref_chosen": -58.33738327026367, + "logps/ref_rejected": -78.31776428222656, + "logps/rejected": -227.2299041748047, + "loss": 1.1075, + "margin_dpo/margin_mean": 42.1923713684082, + "margin_dpo/margin_std": 66.1875, + "step": 565 + }, + { + "KL/chosen_KL_mean": -94.88896179199219, + "KL/mean": -117.41998291015625, + "KL/rejected_KL_mean": -139.95101928710938, + "KL/std": 66.64552307128906, + "epoch": 0.8556311413454271, + "fcm_dpo/beta": 0.010273216292262077, + "fcm_dpo/delta": -0.06323015689849854, + "fcm_dpo/margin": 45.06205749511719, + "fcm_dpo/q_t": 0.39939218759536743, + "grad_norm": 16.0157413482666, + "learning_rate": 3.1537655732553764e-08, + "logits/chosen": 0.6832257509231567, + "logits/rejected": 0.6643606424331665, + "logps/chosen": -166.11270141601562, + "logps/ref_chosen": -71.22373962402344, + "logps/ref_rejected": -71.11601257324219, + "logps/rejected": -211.0670166015625, + "loss": 1.134, + "margin_dpo/margin_mean": 45.06205749511719, + "margin_dpo/margin_std": 80.92869567871094, + "step": 566 + }, + { + "KL/chosen_KL_mean": -93.14676666259766, + "KL/mean": -118.18899536132812, + "KL/rejected_KL_mean": -143.23121643066406, + "KL/std": 65.98379516601562, + "epoch": 0.8571428571428571, + "fcm_dpo/beta": 0.01024097204208374, + "fcm_dpo/delta": -0.1136893481016159, + "fcm_dpo/margin": 50.08445739746094, + "fcm_dpo/q_t": 0.3865007162094116, + "grad_norm": 12.474194526672363, + "learning_rate": 3.089785553471233e-08, + "logits/chosen": 0.6904678344726562, + "logits/rejected": 0.588405430316925, + "logps/chosen": -145.8160400390625, + "logps/ref_chosen": -52.669273376464844, + "logps/ref_rejected": -74.34785461425781, + "logps/rejected": -217.57907104492188, + "loss": 1.0425, + "margin_dpo/margin_mean": 50.084449768066406, + "margin_dpo/margin_std": 64.66056823730469, + "step": 567 + }, + { + "KL/chosen_KL_mean": -87.33174896240234, + "KL/mean": -117.06298065185547, + "KL/rejected_KL_mean": -146.79421997070312, + "KL/std": 67.28833770751953, + "epoch": 0.8586545729402872, + "fcm_dpo/beta": 0.010216941125690937, + "fcm_dpo/delta": -0.20874018967151642, + "fcm_dpo/margin": 59.46246337890625, + "fcm_dpo/q_t": 0.3683282136917114, + "grad_norm": 13.367298126220703, + "learning_rate": 3.026418409484513e-08, + "logits/chosen": 0.6845474243164062, + "logits/rejected": 0.5917050838470459, + "logps/chosen": -139.50975036621094, + "logps/ref_chosen": -52.178001403808594, + "logps/ref_rejected": -85.8277587890625, + "logps/rejected": -232.62197875976562, + "loss": 0.9767, + "margin_dpo/margin_mean": 59.46246337890625, + "margin_dpo/margin_std": 66.27963256835938, + "step": 568 + }, + { + "KL/chosen_KL_mean": -98.3801498413086, + "KL/mean": -117.08673095703125, + "KL/rejected_KL_mean": -135.79330444335938, + "KL/std": 67.76852416992188, + "epoch": 0.8601662887377173, + "fcm_dpo/beta": 0.010184389539062977, + "fcm_dpo/delta": -0.07675791531801224, + "fcm_dpo/margin": 37.41316223144531, + "fcm_dpo/q_t": 0.41637125611305237, + "grad_norm": 14.791778564453125, + "learning_rate": 2.963665913810451e-08, + "logits/chosen": 0.56072598695755, + "logits/rejected": 0.5271100401878357, + "logps/chosen": -161.0294189453125, + "logps/ref_chosen": -62.649261474609375, + "logps/ref_rejected": -75.4298324584961, + "logps/rejected": -211.22314453125, + "loss": 1.1546, + "margin_dpo/margin_mean": 37.41316223144531, + "margin_dpo/margin_std": 66.971435546875, + "step": 569 + }, + { + "KL/chosen_KL_mean": -87.4991455078125, + "KL/mean": -120.22628784179688, + "KL/rejected_KL_mean": -152.95343017578125, + "KL/std": 64.88909912109375, + "epoch": 0.8616780045351474, + "fcm_dpo/beta": 0.0101508479565382, + "fcm_dpo/delta": -0.26609697937965393, + "fcm_dpo/margin": 65.45428466796875, + "fcm_dpo/q_t": 0.35533443093299866, + "grad_norm": 12.467880249023438, + "learning_rate": 2.9015298217712453e-08, + "logits/chosen": 0.6043928861618042, + "logits/rejected": 0.5180938243865967, + "logps/chosen": -137.5409393310547, + "logps/ref_chosen": -50.04179382324219, + "logps/ref_rejected": -78.27146911621094, + "logps/rejected": -231.22488403320312, + "loss": 0.9428, + "margin_dpo/margin_mean": 65.45428466796875, + "margin_dpo/margin_std": 69.83598327636719, + "step": 570 + }, + { + "KL/chosen_KL_mean": -96.78938293457031, + "KL/mean": -115.44367980957031, + "KL/rejected_KL_mean": -134.0979766845703, + "KL/std": 61.940025329589844, + "epoch": 0.8631897203325775, + "fcm_dpo/beta": 0.010139824822545052, + "fcm_dpo/delta": 0.021691590547561646, + "fcm_dpo/margin": 37.30860137939453, + "fcm_dpo/q_t": 0.41338637471199036, + "grad_norm": 14.13100528717041, + "learning_rate": 2.840011871446962e-08, + "logits/chosen": 0.6397497653961182, + "logits/rejected": 0.6060479283332825, + "logps/chosen": -150.44619750976562, + "logps/ref_chosen": -53.65681457519531, + "logps/ref_rejected": -66.13298034667969, + "logps/rejected": -200.23095703125, + "loss": 1.1575, + "margin_dpo/margin_mean": 37.30860137939453, + "margin_dpo/margin_std": 66.62196350097656, + "step": 571 + }, + { + "KL/chosen_KL_mean": -94.654296875, + "KL/mean": -117.70896911621094, + "KL/rejected_KL_mean": -140.76364135742188, + "KL/std": 64.84223937988281, + "epoch": 0.8647014361300076, + "fcm_dpo/beta": 0.010126926004886627, + "fcm_dpo/delta": -0.06725534796714783, + "fcm_dpo/margin": 46.10934829711914, + "fcm_dpo/q_t": 0.39482420682907104, + "grad_norm": 15.302971839904785, + "learning_rate": 2.7791137836269158e-08, + "logits/chosen": 0.6494331359863281, + "logits/rejected": 0.695789098739624, + "logps/chosen": -169.47222900390625, + "logps/ref_chosen": -74.81792449951172, + "logps/ref_rejected": -65.88681030273438, + "logps/rejected": -206.65045166015625, + "loss": 1.0595, + "margin_dpo/margin_mean": 46.109344482421875, + "margin_dpo/margin_std": 60.34523010253906, + "step": 572 + }, + { + "KL/chosen_KL_mean": -105.70700073242188, + "KL/mean": -128.0741729736328, + "KL/rejected_KL_mean": -150.44134521484375, + "KL/std": 70.68563842773438, + "epoch": 0.8662131519274376, + "fcm_dpo/beta": 0.01011162344366312, + "fcm_dpo/delta": -0.05258692800998688, + "fcm_dpo/margin": 44.73434066772461, + "fcm_dpo/q_t": 0.4027160108089447, + "grad_norm": 15.018274307250977, + "learning_rate": 2.718837261761528e-08, + "logits/chosen": 0.6293043494224548, + "logits/rejected": 0.579364538192749, + "logps/chosen": -174.43264770507812, + "logps/ref_chosen": -68.72564697265625, + "logps/ref_rejected": -88.16201782226562, + "logps/rejected": -238.60336303710938, + "loss": 1.1358, + "margin_dpo/margin_mean": 44.734336853027344, + "margin_dpo/margin_std": 81.40330505371094, + "step": 573 + }, + { + "KL/chosen_KL_mean": -92.26235961914062, + "KL/mean": -120.40902709960938, + "KL/rejected_KL_mean": -148.55567932128906, + "KL/std": 65.23068237304688, + "epoch": 0.8677248677248677, + "fcm_dpo/beta": 0.010089917108416557, + "fcm_dpo/delta": -0.16892001032829285, + "fcm_dpo/margin": 56.293312072753906, + "fcm_dpo/q_t": 0.37290358543395996, + "grad_norm": 11.514815330505371, + "learning_rate": 2.659183991914696e-08, + "logits/chosen": 0.6939659714698792, + "logits/rejected": 0.6195951700210571, + "logps/chosen": -148.5757598876953, + "logps/ref_chosen": -56.31340026855469, + "logps/ref_rejected": -83.91553497314453, + "logps/rejected": -232.47122192382812, + "loss": 0.9803, + "margin_dpo/margin_mean": 56.293312072753906, + "margin_dpo/margin_std": 59.513671875, + "step": 574 + }, + { + "KL/chosen_KL_mean": -95.21247100830078, + "KL/mean": -114.33425903320312, + "KL/rejected_KL_mean": -133.45603942871094, + "KL/std": 65.32652282714844, + "epoch": 0.8692365835222978, + "fcm_dpo/beta": 0.01005999743938446, + "fcm_dpo/delta": -0.08129619807004929, + "fcm_dpo/margin": 38.243568420410156, + "fcm_dpo/q_t": 0.4172743260860443, + "grad_norm": 14.525381088256836, + "learning_rate": 2.600155642716606e-08, + "logits/chosen": 0.7005234956741333, + "logits/rejected": 0.6110953092575073, + "logps/chosen": -159.79660034179688, + "logps/ref_chosen": -64.5841293334961, + "logps/ref_rejected": -93.47034454345703, + "logps/rejected": -226.9263916015625, + "loss": 1.1769, + "margin_dpo/margin_mean": 38.243568420410156, + "margin_dpo/margin_std": 74.80801391601562, + "step": 575 + }, + { + "KL/chosen_KL_mean": -84.3938217163086, + "KL/mean": -112.60458374023438, + "KL/rejected_KL_mean": -140.8153533935547, + "KL/std": 66.93716430664062, + "epoch": 0.8707482993197279, + "fcm_dpo/beta": 0.010028736665844917, + "fcm_dpo/delta": -0.1669570505619049, + "fcm_dpo/margin": 56.421539306640625, + "fcm_dpo/q_t": 0.3766467273235321, + "grad_norm": 14.229898452758789, + "learning_rate": 2.5417538653170754e-08, + "logits/chosen": 0.683309018611908, + "logits/rejected": 0.5650166273117065, + "logps/chosen": -137.67434692382812, + "logps/ref_chosen": -53.28052520751953, + "logps/ref_rejected": -84.2000503540039, + "logps/rejected": -225.01541137695312, + "loss": 1.0198, + "margin_dpo/margin_mean": 56.421539306640625, + "margin_dpo/margin_std": 72.51468658447266, + "step": 576 + }, + { + "KL/chosen_KL_mean": -93.55409240722656, + "KL/mean": -114.15921783447266, + "KL/rejected_KL_mean": -134.76434326171875, + "KL/std": 63.764495849609375, + "epoch": 0.872260015117158, + "fcm_dpo/beta": 0.010022887960076332, + "fcm_dpo/delta": -0.013107143342494965, + "fcm_dpo/margin": 41.21025848388672, + "fcm_dpo/q_t": 0.407580703496933, + "grad_norm": 13.894492149353027, + "learning_rate": 2.4839802933393607e-08, + "logits/chosen": 0.6490939855575562, + "logits/rejected": 0.6322671175003052, + "logps/chosen": -155.8787841796875, + "logps/ref_chosen": -62.32468795776367, + "logps/ref_rejected": -67.300537109375, + "logps/rejected": -202.06488037109375, + "loss": 1.1364, + "margin_dpo/margin_mean": 41.21025848388672, + "margin_dpo/margin_std": 71.3342514038086, + "step": 577 + }, + { + "KL/chosen_KL_mean": -90.33680725097656, + "KL/mean": -108.58171844482422, + "KL/rejected_KL_mean": -126.8266372680664, + "KL/std": 65.76661682128906, + "epoch": 0.873771730914588, + "fcm_dpo/beta": 0.01002716924995184, + "fcm_dpo/delta": 0.034235138446092606, + "fcm_dpo/margin": 36.489837646484375, + "fcm_dpo/q_t": 0.4201527237892151, + "grad_norm": 13.627220153808594, + "learning_rate": 2.4268365428344733e-08, + "logits/chosen": 0.7258093357086182, + "logits/rejected": 0.7004756927490234, + "logps/chosen": -146.99237060546875, + "logps/ref_chosen": -56.65557861328125, + "logps/ref_rejected": -68.21835327148438, + "logps/rejected": -195.04498291015625, + "loss": 1.166, + "margin_dpo/margin_mean": 36.489837646484375, + "margin_dpo/margin_std": 69.40409088134766, + "step": 578 + }, + { + "KL/chosen_KL_mean": -94.9715347290039, + "KL/mean": -124.28912353515625, + "KL/rejected_KL_mean": -153.60667419433594, + "KL/std": 64.76129913330078, + "epoch": 0.8752834467120182, + "fcm_dpo/beta": 0.010001586750149727, + "fcm_dpo/delta": -0.18756431341171265, + "fcm_dpo/margin": 58.635154724121094, + "fcm_dpo/q_t": 0.36861079931259155, + "grad_norm": 12.37260913848877, + "learning_rate": 2.3703242122359357e-08, + "logits/chosen": 0.600267231464386, + "logits/rejected": 0.5720463991165161, + "logps/chosen": -151.78118896484375, + "logps/ref_chosen": -56.809661865234375, + "logps/ref_rejected": -68.09613037109375, + "logps/rejected": -221.70281982421875, + "loss": 0.9769, + "margin_dpo/margin_mean": 58.635154724121094, + "margin_dpo/margin_std": 64.10205841064453, + "step": 579 + }, + { + "KL/chosen_KL_mean": -94.31793212890625, + "KL/mean": -116.28801727294922, + "KL/rejected_KL_mean": -138.25808715820312, + "KL/std": 70.97718811035156, + "epoch": 0.8767951625094482, + "fcm_dpo/beta": 0.009991943836212158, + "fcm_dpo/delta": -0.03924529254436493, + "fcm_dpo/margin": 43.940155029296875, + "fcm_dpo/q_t": 0.4046891927719116, + "grad_norm": 14.566301345825195, + "learning_rate": 2.3144448823151392e-08, + "logits/chosen": 0.6197726726531982, + "logits/rejected": 0.5570849180221558, + "logps/chosen": -152.01805114746094, + "logps/ref_chosen": -57.70011520385742, + "logps/ref_rejected": -77.90664672851562, + "logps/rejected": -216.16473388671875, + "loss": 1.1331, + "margin_dpo/margin_mean": 43.940155029296875, + "margin_dpo/margin_std": 77.45632934570312, + "step": 580 + }, + { + "KL/chosen_KL_mean": -105.83895874023438, + "KL/mean": -126.47624969482422, + "KL/rejected_KL_mean": -147.11354064941406, + "KL/std": 64.00870513916016, + "epoch": 0.8783068783068783, + "fcm_dpo/beta": 0.009978841990232468, + "fcm_dpo/delta": -0.0119711235165596, + "fcm_dpo/margin": 41.27458572387695, + "fcm_dpo/q_t": 0.40945544838905334, + "grad_norm": 16.142988204956055, + "learning_rate": 2.259200116137039e-08, + "logits/chosen": 0.6662085056304932, + "logits/rejected": 0.5921996831893921, + "logps/chosen": -165.1713104248047, + "logps/ref_chosen": -59.332359313964844, + "logps/ref_rejected": -83.64482116699219, + "logps/rejected": -230.75836181640625, + "loss": 1.1342, + "margin_dpo/margin_mean": 41.27458572387695, + "margin_dpo/margin_std": 71.12722778320312, + "step": 581 + }, + { + "KL/chosen_KL_mean": -93.78433227539062, + "KL/mean": -117.67637634277344, + "KL/rejected_KL_mean": -141.5684356689453, + "KL/std": 65.80780029296875, + "epoch": 0.8798185941043084, + "fcm_dpo/beta": 0.009972944855690002, + "fcm_dpo/delta": -0.07690756767988205, + "fcm_dpo/margin": 47.78409194946289, + "fcm_dpo/q_t": 0.39471620321273804, + "grad_norm": 14.08938980102539, + "learning_rate": 2.204591459016525e-08, + "logits/chosen": 0.6620016098022461, + "logits/rejected": 0.6906019449234009, + "logps/chosen": -157.9471893310547, + "logps/ref_chosen": -64.16285705566406, + "logps/ref_rejected": -58.632896423339844, + "logps/rejected": -200.20132446289062, + "loss": 1.0798, + "margin_dpo/margin_mean": 47.784088134765625, + "margin_dpo/margin_std": 70.85916900634766, + "step": 582 + }, + { + "KL/chosen_KL_mean": -93.23574829101562, + "KL/mean": -119.99256896972656, + "KL/rejected_KL_mean": -146.74940490722656, + "KL/std": 68.21437072753906, + "epoch": 0.8813303099017384, + "fcm_dpo/beta": 0.00994800589978695, + "fcm_dpo/delta": -0.1330588012933731, + "fcm_dpo/margin": 53.51365661621094, + "fcm_dpo/q_t": 0.38339143991470337, + "grad_norm": 15.661633491516113, + "learning_rate": 2.1506204384751064e-08, + "logits/chosen": 0.7425761818885803, + "logits/rejected": 0.6227637529373169, + "logps/chosen": -145.10813903808594, + "logps/ref_chosen": -51.87239456176758, + "logps/ref_rejected": -83.86331176757812, + "logps/rejected": -230.61270141601562, + "loss": 1.0649, + "margin_dpo/margin_mean": 53.51365661621094, + "margin_dpo/margin_std": 79.15829467773438, + "step": 583 + }, + { + "KL/chosen_KL_mean": -84.57241821289062, + "KL/mean": -107.99671936035156, + "KL/rejected_KL_mean": -131.42098999023438, + "KL/std": 62.97252655029297, + "epoch": 0.8828420256991686, + "fcm_dpo/beta": 0.009926128201186657, + "fcm_dpo/delta": -0.06541818380355835, + "fcm_dpo/margin": 46.84857177734375, + "fcm_dpo/q_t": 0.3998023569583893, + "grad_norm": 14.214855194091797, + "learning_rate": 2.09728856419826e-08, + "logits/chosen": 0.7676564455032349, + "logits/rejected": 0.6545187830924988, + "logps/chosen": -131.14381408691406, + "logps/ref_chosen": -46.571388244628906, + "logps/ref_rejected": -80.67969512939453, + "logps/rejected": -212.10069274902344, + "loss": 1.1202, + "margin_dpo/margin_mean": 46.84857177734375, + "margin_dpo/margin_std": 79.884033203125, + "step": 584 + }, + { + "KL/chosen_KL_mean": -98.87574005126953, + "KL/mean": -116.08522033691406, + "KL/rejected_KL_mean": -133.29470825195312, + "KL/std": 69.75030517578125, + "epoch": 0.8843537414965986, + "fcm_dpo/beta": 0.00994110107421875, + "fcm_dpo/delta": 0.05797140300273895, + "fcm_dpo/margin": 34.418975830078125, + "fcm_dpo/q_t": 0.4221663475036621, + "grad_norm": 13.749168395996094, + "learning_rate": 2.044597327993153e-08, + "logits/chosen": 0.6046024560928345, + "logits/rejected": 0.556342601776123, + "logps/chosen": -157.00027465820312, + "logps/ref_chosen": -58.124534606933594, + "logps/ref_rejected": -79.00538635253906, + "logps/rejected": -212.3000946044922, + "loss": 1.1819, + "margin_dpo/margin_mean": 34.41897201538086, + "margin_dpo/margin_std": 67.51402282714844, + "step": 585 + }, + { + "KL/chosen_KL_mean": -93.72472381591797, + "KL/mean": -118.83309936523438, + "KL/rejected_KL_mean": -143.94146728515625, + "KL/std": 63.781944274902344, + "epoch": 0.8858654572940288, + "fcm_dpo/beta": 0.009922193363308907, + "fcm_dpo/delta": -0.09879402816295624, + "fcm_dpo/margin": 50.21675109863281, + "fcm_dpo/q_t": 0.38783031702041626, + "grad_norm": 15.205615997314453, + "learning_rate": 1.9925482037469187e-08, + "logits/chosen": 0.6866369247436523, + "logits/rejected": 0.6354236602783203, + "logps/chosen": -147.82635498046875, + "logps/ref_chosen": -54.10163879394531, + "logps/ref_rejected": -63.72113037109375, + "logps/rejected": -207.66259765625, + "loss": 1.0286, + "margin_dpo/margin_mean": 50.21675109863281, + "margin_dpo/margin_std": 59.41362380981445, + "step": 586 + }, + { + "KL/chosen_KL_mean": -93.00349426269531, + "KL/mean": -118.10850524902344, + "KL/rejected_KL_mean": -143.21351623535156, + "KL/std": 68.28050231933594, + "epoch": 0.8873771730914588, + "fcm_dpo/beta": 0.009904706850647926, + "fcm_dpo/delta": -0.09781035035848618, + "fcm_dpo/margin": 50.21002197265625, + "fcm_dpo/q_t": 0.38801252841949463, + "grad_norm": 14.259090423583984, + "learning_rate": 1.9411426473854687e-08, + "logits/chosen": 0.6863682270050049, + "logits/rejected": 0.6792365312576294, + "logps/chosen": -156.42068481445312, + "logps/ref_chosen": -63.41719436645508, + "logps/ref_rejected": -63.47003936767578, + "logps/rejected": -206.68356323242188, + "loss": 1.1139, + "margin_dpo/margin_mean": 50.21002197265625, + "margin_dpo/margin_std": 86.71572875976562, + "step": 587 + }, + { + "KL/chosen_KL_mean": -97.0659408569336, + "KL/mean": -123.43115997314453, + "KL/rejected_KL_mean": -149.79637145996094, + "KL/std": 65.96186828613281, + "epoch": 0.8888888888888888, + "fcm_dpo/beta": 0.0098800640553236, + "fcm_dpo/delta": -0.12165706604719162, + "fcm_dpo/margin": 52.730430603027344, + "fcm_dpo/q_t": 0.38594549894332886, + "grad_norm": 16.604976654052734, + "learning_rate": 1.890382096832699e-08, + "logits/chosen": 0.7073137760162354, + "logits/rejected": 0.6585639715194702, + "logps/chosen": -159.2669677734375, + "logps/ref_chosen": -62.20103454589844, + "logps/ref_rejected": -82.10249328613281, + "logps/rejected": -231.89886474609375, + "loss": 1.0519, + "margin_dpo/margin_mean": 52.730430603027344, + "margin_dpo/margin_std": 73.57496643066406, + "step": 588 + }, + { + "KL/chosen_KL_mean": -89.95905303955078, + "KL/mean": -117.0484390258789, + "KL/rejected_KL_mean": -144.1378173828125, + "KL/std": 67.34142303466797, + "epoch": 0.890400604686319, + "fcm_dpo/beta": 0.009858168661594391, + "fcm_dpo/delta": -0.13480722904205322, + "fcm_dpo/margin": 54.178775787353516, + "fcm_dpo/q_t": 0.3800228238105774, + "grad_norm": 12.887765884399414, + "learning_rate": 1.840267971970344e-08, + "logits/chosen": 0.614084005355835, + "logits/rejected": 0.5839520692825317, + "logps/chosen": -146.67266845703125, + "logps/ref_chosen": -56.71361541748047, + "logps/ref_rejected": -76.7366943359375, + "logps/rejected": -220.87451171875, + "loss": 1.0031, + "margin_dpo/margin_mean": 54.178775787353516, + "margin_dpo/margin_std": 60.7816276550293, + "step": 589 + }, + { + "KL/chosen_KL_mean": -100.42595672607422, + "KL/mean": -127.34902954101562, + "KL/rejected_KL_mean": -154.2720947265625, + "KL/std": 63.637847900390625, + "epoch": 0.891912320483749, + "fcm_dpo/beta": 0.009832695126533508, + "fcm_dpo/delta": -0.13012386858463287, + "fcm_dpo/margin": 53.84614181518555, + "fcm_dpo/q_t": 0.3822367787361145, + "grad_norm": 12.990666389465332, + "learning_rate": 1.7908016745981856e-08, + "logits/chosen": 0.6160021424293518, + "logits/rejected": 0.5849137902259827, + "logps/chosen": -166.9397735595703, + "logps/ref_chosen": -66.5138168334961, + "logps/ref_rejected": -85.70820617675781, + "logps/rejected": -239.98031616210938, + "loss": 1.0396, + "margin_dpo/margin_mean": 53.84614181518555, + "margin_dpo/margin_std": 71.66239929199219, + "step": 590 + }, + { + "KL/chosen_KL_mean": -84.79421997070312, + "KL/mean": -116.21773529052734, + "KL/rejected_KL_mean": -147.64125061035156, + "KL/std": 70.36859130859375, + "epoch": 0.8934240362811792, + "fcm_dpo/beta": 0.009784555062651634, + "fcm_dpo/delta": -0.21656835079193115, + "fcm_dpo/margin": 62.84703063964844, + "fcm_dpo/q_t": 0.3677278161048889, + "grad_norm": 16.046977996826172, + "learning_rate": 1.7419845883949098e-08, + "logits/chosen": 0.7307531237602234, + "logits/rejected": 0.6607656478881836, + "logps/chosen": -145.49139404296875, + "logps/ref_chosen": -60.697181701660156, + "logps/ref_rejected": -86.12278747558594, + "logps/rejected": -233.7640380859375, + "loss": 1.0145, + "margin_dpo/margin_mean": 62.84703063964844, + "margin_dpo/margin_std": 82.40460968017578, + "step": 591 + }, + { + "KL/chosen_KL_mean": -88.1783447265625, + "KL/mean": -112.90154266357422, + "KL/rejected_KL_mean": -137.62474060058594, + "KL/std": 63.22344970703125, + "epoch": 0.8949357520786092, + "fcm_dpo/beta": 0.009762782603502274, + "fcm_dpo/delta": -0.08329421281814575, + "fcm_dpo/margin": 49.446388244628906, + "fcm_dpo/q_t": 0.39537298679351807, + "grad_norm": 14.505647659301758, + "learning_rate": 1.6938180788793556e-08, + "logits/chosen": 0.7122167348861694, + "logits/rejected": 0.5920882821083069, + "logps/chosen": -139.41567993164062, + "logps/ref_chosen": -51.237327575683594, + "logps/ref_rejected": -81.60242462158203, + "logps/rejected": -219.2271728515625, + "loss": 1.0665, + "margin_dpo/margin_mean": 49.446388244628906, + "margin_dpo/margin_std": 68.65821075439453, + "step": 592 + }, + { + "KL/chosen_KL_mean": -80.93053436279297, + "KL/mean": -104.6878433227539, + "KL/rejected_KL_mean": -128.44515991210938, + "KL/std": 66.7440185546875, + "epoch": 0.8964474678760394, + "fcm_dpo/beta": 0.00975450687110424, + "fcm_dpo/delta": -0.06379519402980804, + "fcm_dpo/margin": 47.514617919921875, + "fcm_dpo/q_t": 0.39791133999824524, + "grad_norm": 14.989059448242188, + "learning_rate": 1.6463034933723336e-08, + "logits/chosen": 0.6833750009536743, + "logits/rejected": 0.5821329355239868, + "logps/chosen": -123.01053619384766, + "logps/ref_chosen": -42.08000183105469, + "logps/ref_rejected": -68.47499084472656, + "logps/rejected": -196.92013549804688, + "loss": 1.1078, + "margin_dpo/margin_mean": 47.514617919921875, + "margin_dpo/margin_std": 77.56784057617188, + "step": 593 + }, + { + "KL/chosen_KL_mean": -99.91313171386719, + "KL/mean": -121.8230209350586, + "KL/rejected_KL_mean": -143.73291015625, + "KL/std": 65.42422485351562, + "epoch": 0.8979591836734694, + "fcm_dpo/beta": 0.009748319163918495, + "fcm_dpo/delta": -0.027289636433124542, + "fcm_dpo/margin": 43.819786071777344, + "fcm_dpo/q_t": 0.4032715857028961, + "grad_norm": 13.661664009094238, + "learning_rate": 1.5994421609589385e-08, + "logits/chosen": 0.5917218923568726, + "logits/rejected": 0.57381272315979, + "logps/chosen": -163.57180786132812, + "logps/ref_chosen": -63.658668518066406, + "logps/ref_rejected": -70.35597229003906, + "logps/rejected": -214.08889770507812, + "loss": 1.0969, + "margin_dpo/margin_mean": 43.819786071777344, + "margin_dpo/margin_std": 64.7508773803711, + "step": 594 + }, + { + "KL/chosen_KL_mean": -90.78660583496094, + "KL/mean": -120.33202362060547, + "KL/rejected_KL_mean": -149.87744140625, + "KL/std": 67.60763549804688, + "epoch": 0.8994708994708994, + "fcm_dpo/beta": 0.00972592830657959, + "fcm_dpo/delta": -0.17568525671958923, + "fcm_dpo/margin": 59.09083938598633, + "fcm_dpo/q_t": 0.3744524121284485, + "grad_norm": 13.082937240600586, + "learning_rate": 1.553235392451377e-08, + "logits/chosen": 0.7416651248931885, + "logits/rejected": 0.6435179710388184, + "logps/chosen": -147.00537109375, + "logps/ref_chosen": -56.21875762939453, + "logps/ref_rejected": -83.95773315429688, + "logps/rejected": -233.83517456054688, + "loss": 1.0254, + "margin_dpo/margin_mean": 59.090843200683594, + "margin_dpo/margin_std": 78.9394302368164, + "step": 595 + }, + { + "KL/chosen_KL_mean": -103.61746215820312, + "KL/mean": -118.50395202636719, + "KL/rejected_KL_mean": -133.3904266357422, + "KL/std": 63.10828399658203, + "epoch": 0.9009826152683296, + "fcm_dpo/beta": 0.009716038592159748, + "fcm_dpo/delta": 0.012393103912472725, + "fcm_dpo/margin": 29.772972106933594, + "fcm_dpo/q_t": 0.4372863173484802, + "grad_norm": 12.789466857910156, + "learning_rate": 1.507684480352292e-08, + "logits/chosen": 0.5636305809020996, + "logits/rejected": 0.583214282989502, + "logps/chosen": -172.0983428955078, + "logps/ref_chosen": -68.48088073730469, + "logps/ref_rejected": -61.732967376708984, + "logps/rejected": -195.12339782714844, + "loss": 1.2202, + "margin_dpo/margin_mean": 29.772972106933594, + "margin_dpo/margin_std": 68.05734252929688, + "step": 596 + }, + { + "KL/chosen_KL_mean": -80.75653076171875, + "KL/mean": -103.11546325683594, + "KL/rejected_KL_mean": -125.47439575195312, + "KL/std": 58.35403823852539, + "epoch": 0.9024943310657596, + "fcm_dpo/beta": 0.00971193052828312, + "fcm_dpo/delta": -0.03444497659802437, + "fcm_dpo/margin": 44.71786880493164, + "fcm_dpo/q_t": 0.4042968451976776, + "grad_norm": 11.572566986083984, + "learning_rate": 1.4627906988186111e-08, + "logits/chosen": 0.6293044090270996, + "logits/rejected": 0.6105597019195557, + "logps/chosen": -129.61404418945312, + "logps/ref_chosen": -48.85750961303711, + "logps/ref_rejected": -55.068084716796875, + "logps/rejected": -180.54248046875, + "loss": 1.0996, + "margin_dpo/margin_mean": 44.71786880493164, + "margin_dpo/margin_std": 68.23869323730469, + "step": 597 + }, + { + "KL/chosen_KL_mean": -107.01869201660156, + "KL/mean": -121.59184265136719, + "KL/rejected_KL_mean": -136.16497802734375, + "KL/std": 66.391845703125, + "epoch": 0.9040060468631897, + "fcm_dpo/beta": 0.009711449965834618, + "fcm_dpo/delta": 0.021660491824150085, + "fcm_dpo/margin": 29.14628028869629, + "fcm_dpo/q_t": 0.4375370740890503, + "grad_norm": 18.296842575073242, + "learning_rate": 1.4185553036259095e-08, + "logits/chosen": 0.6664835214614868, + "logits/rejected": 0.5825908184051514, + "logps/chosen": -165.90585327148438, + "logps/ref_chosen": -58.88715362548828, + "logps/ref_rejected": -81.43145751953125, + "logps/rejected": -217.596435546875, + "loss": 1.2325, + "margin_dpo/margin_mean": 29.146282196044922, + "margin_dpo/margin_std": 70.06721496582031, + "step": 598 + }, + { + "KL/chosen_KL_mean": -107.95600891113281, + "KL/mean": -125.8758544921875, + "KL/rejected_KL_mean": -143.7957000732422, + "KL/std": 62.69048309326172, + "epoch": 0.9055177626606198, + "fcm_dpo/beta": 0.009723728522658348, + "fcm_dpo/delta": 0.05165565013885498, + "fcm_dpo/margin": 35.839698791503906, + "fcm_dpo/q_t": 0.4230482876300812, + "grad_norm": 14.346553802490234, + "learning_rate": 1.3749795321332885e-08, + "logits/chosen": 0.7184900045394897, + "logits/rejected": 0.6708460450172424, + "logps/chosen": -165.56320190429688, + "logps/ref_chosen": -57.60719299316406, + "logps/ref_rejected": -71.80469512939453, + "logps/rejected": -215.60040283203125, + "loss": 1.1781, + "margin_dpo/margin_mean": 35.839698791503906, + "margin_dpo/margin_std": 70.68020629882812, + "step": 599 + }, + { + "KL/chosen_KL_mean": -97.82275390625, + "KL/mean": -118.94591522216797, + "KL/rejected_KL_mean": -140.069091796875, + "KL/std": 67.70674133300781, + "epoch": 0.9070294784580499, + "fcm_dpo/beta": 0.009730067104101181, + "fcm_dpo/delta": -0.01123511791229248, + "fcm_dpo/margin": 42.24633026123047, + "fcm_dpo/q_t": 0.4093102812767029, + "grad_norm": 15.790849685668945, + "learning_rate": 1.3320646032487393e-08, + "logits/chosen": 0.7301480174064636, + "logits/rejected": 0.6719582080841064, + "logps/chosen": -156.26507568359375, + "logps/ref_chosen": -58.44231414794922, + "logps/ref_rejected": -83.64639282226562, + "logps/rejected": -223.71546936035156, + "loss": 1.129, + "margin_dpo/margin_mean": 42.24633026123047, + "margin_dpo/margin_std": 69.6575927734375, + "step": 600 + }, + { + "KL/chosen_KL_mean": -84.96687316894531, + "KL/mean": -113.00733947753906, + "KL/rejected_KL_mean": -141.0478057861328, + "KL/std": 70.22415924072266, + "epoch": 0.90854119425548, + "fcm_dpo/beta": 0.00969618372619152, + "fcm_dpo/delta": -0.14465658366680145, + "fcm_dpo/margin": 56.08094024658203, + "fcm_dpo/q_t": 0.3816204369068146, + "grad_norm": 13.072766304016113, + "learning_rate": 1.2898117173950868e-08, + "logits/chosen": 0.6706054210662842, + "logits/rejected": 0.5883625745773315, + "logps/chosen": -140.56118774414062, + "logps/ref_chosen": -55.59432601928711, + "logps/ref_rejected": -83.68630981445312, + "logps/rejected": -224.734130859375, + "loss": 1.0463, + "margin_dpo/margin_mean": 56.0809326171875, + "margin_dpo/margin_std": 78.24075317382812, + "step": 601 + }, + { + "KL/chosen_KL_mean": -77.7248306274414, + "KL/mean": -104.5926513671875, + "KL/rejected_KL_mean": -131.46046447753906, + "KL/std": 62.811641693115234, + "epoch": 0.91005291005291, + "fcm_dpo/beta": 0.009678961709141731, + "fcm_dpo/delta": -0.12071502953767776, + "fcm_dpo/margin": 53.73563003540039, + "fcm_dpo/q_t": 0.38376444578170776, + "grad_norm": 12.159416198730469, + "learning_rate": 1.2482220564763667e-08, + "logits/chosen": 0.6899113059043884, + "logits/rejected": 0.657200813293457, + "logps/chosen": -134.0740203857422, + "logps/ref_chosen": -56.349185943603516, + "logps/ref_rejected": -71.9959716796875, + "logps/rejected": -203.45643615722656, + "loss": 1.0253, + "margin_dpo/margin_mean": 53.73563003540039, + "margin_dpo/margin_std": 65.68140411376953, + "step": 602 + }, + { + "KL/chosen_KL_mean": -87.27388000488281, + "KL/mean": -112.95323181152344, + "KL/rejected_KL_mean": -138.63259887695312, + "KL/std": 62.09046173095703, + "epoch": 0.9115646258503401, + "fcm_dpo/beta": 0.009654901921749115, + "fcm_dpo/delta": -0.09635461866855621, + "fcm_dpo/margin": 51.35871887207031, + "fcm_dpo/q_t": 0.3899315595626831, + "grad_norm": 14.848453521728516, + "learning_rate": 1.2072967838448051e-08, + "logits/chosen": 0.6356843709945679, + "logits/rejected": 0.5735263824462891, + "logps/chosen": -140.4422607421875, + "logps/ref_chosen": -53.16838836669922, + "logps/ref_rejected": -73.8604736328125, + "logps/rejected": -212.49307250976562, + "loss": 1.0611, + "margin_dpo/margin_mean": 51.35871887207031, + "margin_dpo/margin_std": 71.87393188476562, + "step": 603 + }, + { + "KL/chosen_KL_mean": -84.29983520507812, + "KL/mean": -107.24708557128906, + "KL/rejected_KL_mean": -130.19432067871094, + "KL/std": 61.57610321044922, + "epoch": 0.9130763416477702, + "fcm_dpo/beta": 0.009646555408835411, + "fcm_dpo/delta": -0.04291178286075592, + "fcm_dpo/margin": 45.89448928833008, + "fcm_dpo/q_t": 0.40309393405914307, + "grad_norm": 14.500489234924316, + "learning_rate": 1.1670370442682459e-08, + "logits/chosen": 0.6216651201248169, + "logits/rejected": 0.6259786486625671, + "logps/chosen": -156.94924926757812, + "logps/ref_chosen": -72.64942169189453, + "logps/ref_rejected": -69.8792724609375, + "logps/rejected": -200.07359313964844, + "loss": 1.1154, + "margin_dpo/margin_mean": 45.894493103027344, + "margin_dpo/margin_std": 76.34507751464844, + "step": 604 + }, + { + "KL/chosen_KL_mean": -99.29381561279297, + "KL/mean": -121.81442260742188, + "KL/rejected_KL_mean": -144.33502197265625, + "KL/std": 63.29581832885742, + "epoch": 0.9145880574452003, + "fcm_dpo/beta": 0.009634751826524734, + "fcm_dpo/delta": -0.03414532542228699, + "fcm_dpo/margin": 45.04121017456055, + "fcm_dpo/q_t": 0.4027559161186218, + "grad_norm": 14.295251846313477, + "learning_rate": 1.1274439638981532e-08, + "logits/chosen": 0.6972644329071045, + "logits/rejected": 0.6373445391654968, + "logps/chosen": -160.9066619873047, + "logps/ref_chosen": -61.61284637451172, + "logps/ref_rejected": -79.34398651123047, + "logps/rejected": -223.67901611328125, + "loss": 1.1202, + "margin_dpo/margin_mean": 45.04120635986328, + "margin_dpo/margin_std": 74.7227554321289, + "step": 605 + }, + { + "KL/chosen_KL_mean": -86.87460327148438, + "KL/mean": -113.5208740234375, + "KL/rejected_KL_mean": -140.16714477539062, + "KL/std": 63.528724670410156, + "epoch": 0.9160997732426304, + "fcm_dpo/beta": 0.009623868390917778, + "fcm_dpo/delta": -0.11344870179891586, + "fcm_dpo/margin": 53.29254913330078, + "fcm_dpo/q_t": 0.3873533308506012, + "grad_norm": 14.767186164855957, + "learning_rate": 1.0885186502381016e-08, + "logits/chosen": 0.6513753533363342, + "logits/rejected": 0.5794322490692139, + "logps/chosen": -141.33883666992188, + "logps/ref_chosen": -54.46424102783203, + "logps/ref_rejected": -79.62708282470703, + "logps/rejected": -219.79421997070312, + "loss": 1.048, + "margin_dpo/margin_mean": 53.292545318603516, + "margin_dpo/margin_std": 72.0080795288086, + "step": 606 + }, + { + "KL/chosen_KL_mean": -95.610595703125, + "KL/mean": -121.06236267089844, + "KL/rejected_KL_mean": -146.51412963867188, + "KL/std": 65.99317169189453, + "epoch": 0.9176114890400605, + "fcm_dpo/beta": 0.009593741036951542, + "fcm_dpo/delta": -0.0890054851770401, + "fcm_dpo/margin": 50.90351867675781, + "fcm_dpo/q_t": 0.3928169012069702, + "grad_norm": 12.503456115722656, + "learning_rate": 1.0502621921127774e-08, + "logits/chosen": 0.6227428913116455, + "logits/rejected": 0.5946848392486572, + "logps/chosen": -158.47146606445312, + "logps/ref_chosen": -62.86086654663086, + "logps/ref_rejected": -72.5501937866211, + "logps/rejected": -219.06431579589844, + "loss": 1.0682, + "margin_dpo/margin_mean": 50.90352249145508, + "margin_dpo/margin_std": 71.05351257324219, + "step": 607 + }, + { + "KL/chosen_KL_mean": -99.68170166015625, + "KL/mean": -122.40928649902344, + "KL/rejected_KL_mean": -145.13685607910156, + "KL/std": 65.85370635986328, + "epoch": 0.9191232048374905, + "fcm_dpo/beta": 0.009592675603926182, + "fcm_dpo/delta": -0.03619318827986717, + "fcm_dpo/margin": 45.455142974853516, + "fcm_dpo/q_t": 0.4029122591018677, + "grad_norm": 14.058253288269043, + "learning_rate": 1.0126756596375685e-08, + "logits/chosen": 0.6333595514297485, + "logits/rejected": 0.5479701161384583, + "logps/chosen": -162.8624267578125, + "logps/ref_chosen": -63.18071746826172, + "logps/ref_rejected": -99.15888214111328, + "logps/rejected": -244.29574584960938, + "loss": 1.0894, + "margin_dpo/margin_mean": 45.455142974853516, + "margin_dpo/margin_std": 66.17314147949219, + "step": 608 + }, + { + "KL/chosen_KL_mean": -84.85282135009766, + "KL/mean": -114.03013610839844, + "KL/rejected_KL_mean": -143.2074432373047, + "KL/std": 65.55744934082031, + "epoch": 0.9206349206349206, + "fcm_dpo/beta": 0.009562736377120018, + "fcm_dpo/delta": -0.15905527770519257, + "fcm_dpo/margin": 58.354618072509766, + "fcm_dpo/q_t": 0.3731173574924469, + "grad_norm": 11.367518424987793, + "learning_rate": 9.757601041885694e-09, + "logits/chosen": 0.7596822381019592, + "logits/rejected": 0.7207983732223511, + "logps/chosen": -133.47604370117188, + "logps/ref_chosen": -48.62322235107422, + "logps/ref_rejected": -68.28271484375, + "logps/rejected": -211.4901580810547, + "loss": 0.994, + "margin_dpo/margin_mean": 58.354618072509766, + "margin_dpo/margin_std": 63.273841857910156, + "step": 609 + }, + { + "KL/chosen_KL_mean": -94.60691833496094, + "KL/mean": -121.34956359863281, + "KL/rejected_KL_mean": -148.09222412109375, + "KL/std": 67.83274841308594, + "epoch": 0.9221466364323507, + "fcm_dpo/beta": 0.009541090577840805, + "fcm_dpo/delta": -0.11093239486217499, + "fcm_dpo/margin": 53.48529052734375, + "fcm_dpo/q_t": 0.38859251141548157, + "grad_norm": 12.606019020080566, + "learning_rate": 9.395165583732379e-09, + "logits/chosen": 0.6452124118804932, + "logits/rejected": 0.6406885385513306, + "logps/chosen": -167.2720489501953, + "logps/ref_chosen": -72.66513061523438, + "logps/ref_rejected": -87.15310668945312, + "logps/rejected": -235.24533081054688, + "loss": 1.0515, + "margin_dpo/margin_mean": 53.485294342041016, + "margin_dpo/margin_std": 73.50416564941406, + "step": 610 + }, + { + "KL/chosen_KL_mean": -86.79630279541016, + "KL/mean": -108.47848510742188, + "KL/rejected_KL_mean": -130.16065979003906, + "KL/std": 62.57762145996094, + "epoch": 0.9236583522297808, + "fcm_dpo/beta": 0.009533722884953022, + "fcm_dpo/delta": -0.013507997617125511, + "fcm_dpo/margin": 43.36436462402344, + "fcm_dpo/q_t": 0.4055609107017517, + "grad_norm": 13.10541820526123, + "learning_rate": 9.03946036001449e-09, + "logits/chosen": 0.6926069259643555, + "logits/rejected": 0.6413847208023071, + "logps/chosen": -135.10488891601562, + "logps/ref_chosen": -48.30857849121094, + "logps/ref_rejected": -70.6141128540039, + "logps/rejected": -200.7747802734375, + "loss": 1.0978, + "margin_dpo/margin_mean": 43.36436462402344, + "margin_dpo/margin_std": 61.92253112792969, + "step": 611 + }, + { + "KL/chosen_KL_mean": -97.07661437988281, + "KL/mean": -125.2576904296875, + "KL/rejected_KL_mean": -153.43875122070312, + "KL/std": 66.92588806152344, + "epoch": 0.9251700680272109, + "fcm_dpo/beta": 0.0095168212428689, + "fcm_dpo/delta": -0.13712337613105774, + "fcm_dpo/margin": 56.36213302612305, + "fcm_dpo/q_t": 0.3803968131542206, + "grad_norm": 12.471902847290039, + "learning_rate": 8.690495320571839e-09, + "logits/chosen": 0.5583537817001343, + "logits/rejected": 0.4868154525756836, + "logps/chosen": -158.30816650390625, + "logps/ref_chosen": -61.23155975341797, + "logps/ref_rejected": -94.37979888916016, + "logps/rejected": -247.8185577392578, + "loss": 1.0305, + "margin_dpo/margin_mean": 56.36212921142578, + "margin_dpo/margin_std": 73.42439270019531, + "step": 612 + }, + { + "KL/chosen_KL_mean": -82.11544799804688, + "KL/mean": -111.40314483642578, + "KL/rejected_KL_mean": -140.6908416748047, + "KL/std": 68.44509887695312, + "epoch": 0.926681783824641, + "fcm_dpo/beta": 0.00949193350970745, + "fcm_dpo/delta": -0.1568354368209839, + "fcm_dpo/margin": 58.57539367675781, + "fcm_dpo/q_t": 0.37585777044296265, + "grad_norm": 11.184490203857422, + "learning_rate": 8.348280226706722e-09, + "logits/chosen": 0.5663385987281799, + "logits/rejected": 0.5627808570861816, + "logps/chosen": -136.0985565185547, + "logps/ref_chosen": -53.98310852050781, + "logps/ref_rejected": -58.32208251953125, + "logps/rejected": -199.01292419433594, + "loss": 1.007, + "margin_dpo/margin_mean": 58.57539367675781, + "margin_dpo/margin_std": 69.83525848388672, + "step": 613 + }, + { + "KL/chosen_KL_mean": -94.65059661865234, + "KL/mean": -121.04035186767578, + "KL/rejected_KL_mean": -147.43011474609375, + "KL/std": 61.2188835144043, + "epoch": 0.9281934996220711, + "fcm_dpo/beta": 0.00947068352252245, + "fcm_dpo/delta": -0.1003570631146431, + "fcm_dpo/margin": 52.779518127441406, + "fcm_dpo/q_t": 0.38562172651290894, + "grad_norm": 13.797905921936035, + "learning_rate": 8.012824650910937e-09, + "logits/chosen": 0.7042652368545532, + "logits/rejected": 0.6946673393249512, + "logps/chosen": -154.8936309814453, + "logps/ref_chosen": -60.24303436279297, + "logps/ref_rejected": -72.26258850097656, + "logps/rejected": -219.6927032470703, + "loss": 1.0347, + "margin_dpo/margin_mean": 52.779518127441406, + "margin_dpo/margin_std": 63.84077835083008, + "step": 614 + }, + { + "KL/chosen_KL_mean": -99.37279510498047, + "KL/mean": -126.70684814453125, + "KL/rejected_KL_mean": -154.04090881347656, + "KL/std": 74.2895278930664, + "epoch": 0.9297052154195011, + "fcm_dpo/beta": 0.009439119137823582, + "fcm_dpo/delta": -0.11676755547523499, + "fcm_dpo/margin": 54.66810989379883, + "fcm_dpo/q_t": 0.3869086503982544, + "grad_norm": 12.691832542419434, + "learning_rate": 7.684137976598088e-09, + "logits/chosen": 0.6369056701660156, + "logits/rejected": 0.5804462432861328, + "logps/chosen": -171.46746826171875, + "logps/ref_chosen": -72.09467315673828, + "logps/ref_rejected": -104.02980041503906, + "logps/rejected": -258.0707092285156, + "loss": 1.0748, + "margin_dpo/margin_mean": 54.668113708496094, + "margin_dpo/margin_std": 82.04415893554688, + "step": 615 + }, + { + "KL/chosen_KL_mean": -94.62140655517578, + "KL/mean": -117.4256591796875, + "KL/rejected_KL_mean": -140.2299041748047, + "KL/std": 68.09626770019531, + "epoch": 0.9312169312169312, + "fcm_dpo/beta": 0.009432796388864517, + "fcm_dpo/delta": -0.03036539815366268, + "fcm_dpo/margin": 45.608497619628906, + "fcm_dpo/q_t": 0.4043644368648529, + "grad_norm": 12.048199653625488, + "learning_rate": 7.36222939784098e-09, + "logits/chosen": 0.7065518498420715, + "logits/rejected": 0.6181278228759766, + "logps/chosen": -153.15213012695312, + "logps/ref_chosen": -58.530723571777344, + "logps/ref_rejected": -75.48025512695312, + "logps/rejected": -215.71014404296875, + "loss": 1.1022, + "margin_dpo/margin_mean": 45.608497619628906, + "margin_dpo/margin_std": 69.74759674072266, + "step": 616 + }, + { + "KL/chosen_KL_mean": -104.14596557617188, + "KL/mean": -124.9691390991211, + "KL/rejected_KL_mean": -145.7923126220703, + "KL/std": 64.73648071289062, + "epoch": 0.9327286470143613, + "fcm_dpo/beta": 0.009426670148968697, + "fcm_dpo/delta": 0.007304169237613678, + "fcm_dpo/margin": 41.6463508605957, + "fcm_dpo/q_t": 0.411568820476532, + "grad_norm": 15.833730697631836, + "learning_rate": 7.047107919114586e-09, + "logits/chosen": 0.6855121850967407, + "logits/rejected": 0.6302813291549683, + "logps/chosen": -161.754638671875, + "logps/ref_chosen": -57.608673095703125, + "logps/ref_rejected": -81.22109985351562, + "logps/rejected": -227.01341247558594, + "loss": 1.129, + "margin_dpo/margin_mean": 41.64635467529297, + "margin_dpo/margin_std": 66.01541137695312, + "step": 617 + }, + { + "KL/chosen_KL_mean": -90.28805541992188, + "KL/mean": -115.227294921875, + "KL/rejected_KL_mean": -140.16653442382812, + "KL/std": 70.46360778808594, + "epoch": 0.9342403628117913, + "fcm_dpo/beta": 0.009421506896615028, + "fcm_dpo/delta": -0.07031630724668503, + "fcm_dpo/margin": 49.87847900390625, + "fcm_dpo/q_t": 0.39561623334884644, + "grad_norm": 15.244210243225098, + "learning_rate": 6.738782355044048e-09, + "logits/chosen": 0.6184258460998535, + "logits/rejected": 0.5102506875991821, + "logps/chosen": -146.9840087890625, + "logps/ref_chosen": -56.69594192504883, + "logps/ref_rejected": -85.92362976074219, + "logps/rejected": -226.09017944335938, + "loss": 1.0591, + "margin_dpo/margin_mean": 49.87847900390625, + "margin_dpo/margin_std": 65.98014831542969, + "step": 618 + }, + { + "KL/chosen_KL_mean": -88.17597198486328, + "KL/mean": -112.64971923828125, + "KL/rejected_KL_mean": -137.12347412109375, + "KL/std": 64.82576751708984, + "epoch": 0.9357520786092215, + "fcm_dpo/beta": 0.009411046281456947, + "fcm_dpo/delta": -0.060951799154281616, + "fcm_dpo/margin": 48.947505950927734, + "fcm_dpo/q_t": 0.396110475063324, + "grad_norm": 13.399476051330566, + "learning_rate": 6.437261330158206e-09, + "logits/chosen": 0.7435458898544312, + "logits/rejected": 0.662152111530304, + "logps/chosen": -142.23439025878906, + "logps/ref_chosen": -54.05841827392578, + "logps/ref_rejected": -83.55493927001953, + "logps/rejected": -220.6784210205078, + "loss": 1.0719, + "margin_dpo/margin_mean": 48.947505950927734, + "margin_dpo/margin_std": 67.86476135253906, + "step": 619 + }, + { + "KL/chosen_KL_mean": -87.03279113769531, + "KL/mean": -108.719970703125, + "KL/rejected_KL_mean": -130.4071502685547, + "KL/std": 61.34162521362305, + "epoch": 0.9372637944066515, + "fcm_dpo/beta": 0.009399937465786934, + "fcm_dpo/delta": -0.11013805121183395, + "fcm_dpo/margin": 43.374366760253906, + "fcm_dpo/q_t": 0.4091581106185913, + "grad_norm": 14.783217430114746, + "learning_rate": 6.142553278648238e-09, + "logits/chosen": 0.6671056151390076, + "logits/rejected": 0.6660356521606445, + "logps/chosen": -150.40249633789062, + "logps/ref_chosen": -63.36971664428711, + "logps/ref_rejected": -65.68269348144531, + "logps/rejected": -196.08984375, + "loss": 1.1157, + "margin_dpo/margin_mean": 43.37437057495117, + "margin_dpo/margin_std": 64.71560668945312, + "step": 620 + }, + { + "KL/chosen_KL_mean": -90.24835205078125, + "KL/mean": -112.16598510742188, + "KL/rejected_KL_mean": -134.0836181640625, + "KL/std": 64.00093841552734, + "epoch": 0.9387755102040817, + "fcm_dpo/beta": 0.009390762075781822, + "fcm_dpo/delta": -0.011709349229931831, + "fcm_dpo/margin": 43.835269927978516, + "fcm_dpo/q_t": 0.4082076847553253, + "grad_norm": 13.780265808105469, + "learning_rate": 5.854666444131934e-09, + "logits/chosen": 0.7073573470115662, + "logits/rejected": 0.5901222229003906, + "logps/chosen": -142.569580078125, + "logps/ref_chosen": -52.321224212646484, + "logps/ref_rejected": -88.09001159667969, + "logps/rejected": -222.17361450195312, + "loss": 1.1206, + "margin_dpo/margin_mean": 43.83527374267578, + "margin_dpo/margin_std": 71.3182144165039, + "step": 621 + }, + { + "KL/chosen_KL_mean": -99.75466918945312, + "KL/mean": -122.72030639648438, + "KL/rejected_KL_mean": -145.68597412109375, + "KL/std": 67.6224365234375, + "epoch": 0.9402872260015117, + "fcm_dpo/beta": 0.009385638870298862, + "fcm_dpo/delta": -0.031236322596669197, + "fcm_dpo/margin": 45.931297302246094, + "fcm_dpo/q_t": 0.4027305841445923, + "grad_norm": 15.391279220581055, + "learning_rate": 5.573608879422875e-09, + "logits/chosen": 0.6228291988372803, + "logits/rejected": 0.578609824180603, + "logps/chosen": -159.6201171875, + "logps/ref_chosen": -59.86545944213867, + "logps/ref_rejected": -81.86668395996094, + "logps/rejected": -227.55264282226562, + "loss": 1.0888, + "margin_dpo/margin_mean": 45.931297302246094, + "margin_dpo/margin_std": 65.02021026611328, + "step": 622 + }, + { + "KL/chosen_KL_mean": -106.66089630126953, + "KL/mean": -131.53106689453125, + "KL/rejected_KL_mean": -156.40121459960938, + "KL/std": 69.8499984741211, + "epoch": 0.9417989417989417, + "fcm_dpo/beta": 0.009369564242661, + "fcm_dpo/delta": -0.06638877093791962, + "fcm_dpo/margin": 49.740318298339844, + "fcm_dpo/q_t": 0.39517927169799805, + "grad_norm": 13.09245777130127, + "learning_rate": 5.299388446305342e-09, + "logits/chosen": 0.6379223465919495, + "logits/rejected": 0.5756089687347412, + "logps/chosen": -174.02935791015625, + "logps/ref_chosen": -67.36846160888672, + "logps/ref_rejected": -82.02733612060547, + "logps/rejected": -238.42855834960938, + "loss": 1.0668, + "margin_dpo/margin_mean": 49.74032211303711, + "margin_dpo/margin_std": 67.55683898925781, + "step": 623 + }, + { + "KL/chosen_KL_mean": -86.39067077636719, + "KL/mean": -113.67745971679688, + "KL/rejected_KL_mean": -140.9642333984375, + "KL/std": 67.56327056884766, + "epoch": 0.9433106575963719, + "fcm_dpo/beta": 0.009351705200970173, + "fcm_dpo/delta": -0.11095096170902252, + "fcm_dpo/margin": 54.573577880859375, + "fcm_dpo/q_t": 0.3884219825267792, + "grad_norm": 13.307804107666016, + "learning_rate": 5.03201281531429e-09, + "logits/chosen": 0.6588263511657715, + "logits/rejected": 0.5565637946128845, + "logps/chosen": -137.41722106933594, + "logps/ref_chosen": -51.02655029296875, + "logps/ref_rejected": -76.49203491210938, + "logps/rejected": -217.45626831054688, + "loss": 1.055, + "margin_dpo/margin_mean": 54.573577880859375, + "margin_dpo/margin_std": 75.72547912597656, + "step": 624 + }, + { + "KL/chosen_KL_mean": -91.36255645751953, + "KL/mean": -111.55204772949219, + "KL/rejected_KL_mean": -131.74154663085938, + "KL/std": 64.22966003417969, + "epoch": 0.9448223733938019, + "fcm_dpo/beta": 0.009351881220936775, + "fcm_dpo/delta": 0.022463934496045113, + "fcm_dpo/margin": 40.37898254394531, + "fcm_dpo/q_t": 0.416697233915329, + "grad_norm": 13.203941345214844, + "learning_rate": 4.7714894655209174e-09, + "logits/chosen": 0.7235609292984009, + "logits/rejected": 0.6313952207565308, + "logps/chosen": -145.57017517089844, + "logps/ref_chosen": -54.20761489868164, + "logps/ref_rejected": -84.93669128417969, + "logps/rejected": -216.67823791503906, + "loss": 1.1567, + "margin_dpo/margin_mean": 40.37898635864258, + "margin_dpo/margin_std": 74.94140625, + "step": 625 + }, + { + "KL/chosen_KL_mean": -86.2386474609375, + "KL/mean": -113.31793975830078, + "KL/rejected_KL_mean": -140.397216796875, + "KL/std": 69.18659973144531, + "epoch": 0.9463340891912321, + "fcm_dpo/beta": 0.009336723014712334, + "fcm_dpo/delta": -0.10621291399002075, + "fcm_dpo/margin": 54.1585807800293, + "fcm_dpo/q_t": 0.3917349576950073, + "grad_norm": 13.739357948303223, + "learning_rate": 4.517825684323323e-09, + "logits/chosen": 0.7683677673339844, + "logits/rejected": 0.6254994869232178, + "logps/chosen": -131.3006591796875, + "logps/ref_chosen": -45.06201934814453, + "logps/ref_rejected": -89.66368103027344, + "logps/rejected": -230.0609130859375, + "loss": 1.0812, + "margin_dpo/margin_mean": 54.15858459472656, + "margin_dpo/margin_std": 83.95677185058594, + "step": 626 + }, + { + "KL/chosen_KL_mean": -95.94741821289062, + "KL/mean": -126.11305236816406, + "KL/rejected_KL_mean": -156.2786865234375, + "KL/std": 68.88520812988281, + "epoch": 0.9478458049886621, + "fcm_dpo/beta": 0.009308360517024994, + "fcm_dpo/delta": -0.16253156960010529, + "fcm_dpo/margin": 60.331268310546875, + "fcm_dpo/q_t": 0.3748932480812073, + "grad_norm": 13.019854545593262, + "learning_rate": 4.271028567242818e-09, + "logits/chosen": 0.6114860773086548, + "logits/rejected": 0.48356151580810547, + "logps/chosen": -154.73846435546875, + "logps/ref_chosen": -58.791053771972656, + "logps/ref_rejected": -94.90802001953125, + "logps/rejected": -251.18670654296875, + "loss": 1.0104, + "margin_dpo/margin_mean": 60.331268310546875, + "margin_dpo/margin_std": 74.0421142578125, + "step": 627 + }, + { + "KL/chosen_KL_mean": -85.78089904785156, + "KL/mean": -115.78858947753906, + "KL/rejected_KL_mean": -145.79629516601562, + "KL/std": 68.70565032958984, + "epoch": 0.9493575207860923, + "fcm_dpo/beta": 0.00929003581404686, + "fcm_dpo/delta": -0.1584821343421936, + "fcm_dpo/margin": 60.01538848876953, + "fcm_dpo/q_t": 0.37497538328170776, + "grad_norm": 15.755816459655762, + "learning_rate": 4.0311050177251895e-09, + "logits/chosen": 0.6575284004211426, + "logits/rejected": 0.6176701784133911, + "logps/chosen": -138.58447265625, + "logps/ref_chosen": -52.80357360839844, + "logps/ref_rejected": -76.49468994140625, + "logps/rejected": -222.29098510742188, + "loss": 1.0396, + "margin_dpo/margin_mean": 60.01538848876953, + "margin_dpo/margin_std": 77.32579040527344, + "step": 628 + }, + { + "KL/chosen_KL_mean": -90.83419799804688, + "KL/mean": -114.13834381103516, + "KL/rejected_KL_mean": -137.4425048828125, + "KL/std": 64.1907730102539, + "epoch": 0.9508692365835223, + "fcm_dpo/beta": 0.009270558133721352, + "fcm_dpo/delta": -0.03222731128334999, + "fcm_dpo/margin": 46.608314514160156, + "fcm_dpo/q_t": 0.4028470516204834, + "grad_norm": 13.420146942138672, + "learning_rate": 3.798061746947995e-09, + "logits/chosen": 0.6638208031654358, + "logits/rejected": 0.6569257974624634, + "logps/chosen": -161.55169677734375, + "logps/ref_chosen": -70.71749877929688, + "logps/ref_rejected": -78.96273803710938, + "logps/rejected": -216.40524291992188, + "loss": 1.0748, + "margin_dpo/margin_mean": 46.608314514160156, + "margin_dpo/margin_std": 60.72931671142578, + "step": 629 + }, + { + "KL/chosen_KL_mean": -90.14724731445312, + "KL/mean": -118.18379211425781, + "KL/rejected_KL_mean": -146.2203369140625, + "KL/std": 70.44136047363281, + "epoch": 0.9523809523809523, + "fcm_dpo/beta": 0.009253723546862602, + "fcm_dpo/delta": -0.11948807537555695, + "fcm_dpo/margin": 56.07310485839844, + "fcm_dpo/q_t": 0.38493478298187256, + "grad_norm": 10.419124603271484, + "learning_rate": 3.5719052736323806e-09, + "logits/chosen": 0.6217161417007446, + "logits/rejected": 0.5728244185447693, + "logps/chosen": -146.34866333007812, + "logps/ref_chosen": -56.201412200927734, + "logps/ref_rejected": -74.69807434082031, + "logps/rejected": -220.91842651367188, + "loss": 1.0318, + "margin_dpo/margin_mean": 56.07310485839844, + "margin_dpo/margin_std": 70.69473266601562, + "step": 630 + }, + { + "KL/chosen_KL_mean": -83.75495910644531, + "KL/mean": -116.14313507080078, + "KL/rejected_KL_mean": -148.53131103515625, + "KL/std": 71.75352478027344, + "epoch": 0.9538926681783825, + "fcm_dpo/beta": 0.009208369068801403, + "fcm_dpo/delta": -0.1980532556772232, + "fcm_dpo/margin": 64.7763442993164, + "fcm_dpo/q_t": 0.371199369430542, + "grad_norm": 13.92465877532959, + "learning_rate": 3.352641923861144e-09, + "logits/chosen": 0.757738471031189, + "logits/rejected": 0.6397895812988281, + "logps/chosen": -142.5755615234375, + "logps/ref_chosen": -58.82059860229492, + "logps/ref_rejected": -96.51437377929688, + "logps/rejected": -245.04568481445312, + "loss": 1.0016, + "margin_dpo/margin_mean": 64.7763442993164, + "margin_dpo/margin_std": 78.53189086914062, + "step": 631 + }, + { + "KL/chosen_KL_mean": -87.59185791015625, + "KL/mean": -118.85031127929688, + "KL/rejected_KL_mean": -150.1087646484375, + "KL/std": 66.44097900390625, + "epoch": 0.9554043839758125, + "fcm_dpo/beta": 0.009185729548335075, + "fcm_dpo/delta": -0.17524564266204834, + "fcm_dpo/margin": 62.51689910888672, + "fcm_dpo/q_t": 0.3709558844566345, + "grad_norm": 12.046740531921387, + "learning_rate": 3.140277830901428e-09, + "logits/chosen": 0.6751530766487122, + "logits/rejected": 0.6538740396499634, + "logps/chosen": -146.37791442871094, + "logps/ref_chosen": -58.786048889160156, + "logps/ref_rejected": -67.21923828125, + "logps/rejected": -217.3280029296875, + "loss": 0.9833, + "margin_dpo/margin_mean": 62.51689910888672, + "margin_dpo/margin_std": 68.6002197265625, + "step": 632 + }, + { + "KL/chosen_KL_mean": -82.53929138183594, + "KL/mean": -105.67863464355469, + "KL/rejected_KL_mean": -128.81796264648438, + "KL/std": 63.09522247314453, + "epoch": 0.9569160997732427, + "fcm_dpo/beta": 0.009175149723887444, + "fcm_dpo/delta": -0.024726668372750282, + "fcm_dpo/margin": 46.27867889404297, + "fcm_dpo/q_t": 0.4061383605003357, + "grad_norm": 13.06899642944336, + "learning_rate": 2.9348189350335007e-09, + "logits/chosen": 0.6440326571464539, + "logits/rejected": 0.579005241394043, + "logps/chosen": -134.66949462890625, + "logps/ref_chosen": -52.13019561767578, + "logps/ref_rejected": -67.23016357421875, + "logps/rejected": -196.04812622070312, + "loss": 1.1053, + "margin_dpo/margin_mean": 46.27867889404297, + "margin_dpo/margin_std": 71.02957916259766, + "step": 633 + }, + { + "KL/chosen_KL_mean": -98.99051666259766, + "KL/mean": -110.6861343383789, + "KL/rejected_KL_mean": -122.38175964355469, + "KL/std": 63.44605255126953, + "epoch": 0.9584278155706727, + "fcm_dpo/beta": 0.009178705513477325, + "fcm_dpo/delta": 0.07809285074472427, + "fcm_dpo/margin": 23.391246795654297, + "fcm_dpo/q_t": 0.45291978120803833, + "grad_norm": 16.723522186279297, + "learning_rate": 2.736270983384276e-09, + "logits/chosen": 0.7354230880737305, + "logits/rejected": 0.7486121654510498, + "logps/chosen": -159.97030639648438, + "logps/ref_chosen": -60.97979736328125, + "logps/ref_rejected": -58.50825119018555, + "logps/rejected": -180.8900146484375, + "loss": 1.2821, + "margin_dpo/margin_mean": 23.391246795654297, + "margin_dpo/margin_std": 70.49496459960938, + "step": 634 + }, + { + "KL/chosen_KL_mean": -112.91930389404297, + "KL/mean": -131.96063232421875, + "KL/rejected_KL_mean": -151.001953125, + "KL/std": 64.30419921875, + "epoch": 0.9599395313681028, + "fcm_dpo/beta": 0.009194991551339626, + "fcm_dpo/delta": 0.049981310963630676, + "fcm_dpo/margin": 38.082672119140625, + "fcm_dpo/q_t": 0.422548770904541, + "grad_norm": 11.76205062866211, + "learning_rate": 2.5446395297668287e-09, + "logits/chosen": 0.566962480545044, + "logits/rejected": 0.49914032220840454, + "logps/chosen": -178.89231872558594, + "logps/ref_chosen": -65.9730224609375, + "logps/ref_rejected": -85.61317443847656, + "logps/rejected": -236.61512756347656, + "loss": 1.1917, + "margin_dpo/margin_mean": 38.08266830444336, + "margin_dpo/margin_std": 79.28173828125, + "step": 635 + }, + { + "KL/chosen_KL_mean": -87.38157653808594, + "KL/mean": -114.48095703125, + "KL/rejected_KL_mean": -141.580322265625, + "KL/std": 66.9657974243164, + "epoch": 0.9614512471655329, + "fcm_dpo/beta": 0.009178088046610355, + "fcm_dpo/delta": -0.09799392521381378, + "fcm_dpo/margin": 54.19874954223633, + "fcm_dpo/q_t": 0.38755351305007935, + "grad_norm": 10.788540840148926, + "learning_rate": 2.359929934524829e-09, + "logits/chosen": 0.6494250297546387, + "logits/rejected": 0.549432098865509, + "logps/chosen": -136.52174377441406, + "logps/ref_chosen": -49.140167236328125, + "logps/ref_rejected": -81.26971435546875, + "logps/rejected": -222.85003662109375, + "loss": 1.0434, + "margin_dpo/margin_mean": 54.198753356933594, + "margin_dpo/margin_std": 69.11613464355469, + "step": 636 + }, + { + "KL/chosen_KL_mean": -116.79802703857422, + "KL/mean": -136.7506103515625, + "KL/rejected_KL_mean": -156.7032012939453, + "KL/std": 62.35839080810547, + "epoch": 0.9629629629629629, + "fcm_dpo/beta": 0.009183991700410843, + "fcm_dpo/delta": 0.03361337259411812, + "fcm_dpo/margin": 39.90517044067383, + "fcm_dpo/q_t": 0.42024946212768555, + "grad_norm": 14.025014877319336, + "learning_rate": 2.1821473643827137e-09, + "logits/chosen": 0.6320610642433167, + "logits/rejected": 0.558428943157196, + "logps/chosen": -190.49461364746094, + "logps/ref_chosen": -73.69658660888672, + "logps/ref_rejected": -83.01487731933594, + "logps/rejected": -239.71807861328125, + "loss": 1.1739, + "margin_dpo/margin_mean": 39.90516662597656, + "margin_dpo/margin_std": 78.98249816894531, + "step": 637 + }, + { + "KL/chosen_KL_mean": -101.84536743164062, + "KL/mean": -125.6198501586914, + "KL/rejected_KL_mean": -149.3943328857422, + "KL/std": 67.13969421386719, + "epoch": 0.9644746787603931, + "fcm_dpo/beta": 0.009179932065308094, + "fcm_dpo/delta": -0.03665650263428688, + "fcm_dpo/margin": 47.54895782470703, + "fcm_dpo/q_t": 0.4027141332626343, + "grad_norm": 13.555511474609375, + "learning_rate": 2.0112967923011646e-09, + "logits/chosen": 0.6586599349975586, + "logits/rejected": 0.6086690425872803, + "logps/chosen": -164.626953125, + "logps/ref_chosen": -62.78158187866211, + "logps/ref_rejected": -85.40478515625, + "logps/rejected": -234.7991180419922, + "loss": 1.0948, + "margin_dpo/margin_mean": 47.54895782470703, + "margin_dpo/margin_std": 70.826416015625, + "step": 638 + }, + { + "KL/chosen_KL_mean": -91.68496704101562, + "KL/mean": -118.87995910644531, + "KL/rejected_KL_mean": -146.074951171875, + "KL/std": 69.62225341796875, + "epoch": 0.9659863945578231, + "fcm_dpo/beta": 0.009165138006210327, + "fcm_dpo/delta": -0.09897308051586151, + "fcm_dpo/margin": 54.38999938964844, + "fcm_dpo/q_t": 0.39029204845428467, + "grad_norm": 13.898356437683105, + "learning_rate": 1.847382997337943e-09, + "logits/chosen": 0.653825044631958, + "logits/rejected": 0.545742928981781, + "logps/chosen": -145.45155334472656, + "logps/ref_chosen": -53.76658630371094, + "logps/ref_rejected": -72.30009460449219, + "logps/rejected": -218.37506103515625, + "loss": 1.048, + "margin_dpo/margin_mean": 54.38999938964844, + "margin_dpo/margin_std": 71.9715576171875, + "step": 639 + }, + { + "KL/chosen_KL_mean": -95.94805145263672, + "KL/mean": -119.98507690429688, + "KL/rejected_KL_mean": -144.0220947265625, + "KL/std": 67.10240173339844, + "epoch": 0.9674981103552532, + "fcm_dpo/beta": 0.009151199832558632, + "fcm_dpo/delta": -0.040125321596860886, + "fcm_dpo/margin": 48.074039459228516, + "fcm_dpo/q_t": 0.399217426776886, + "grad_norm": 12.810059547424316, + "learning_rate": 1.690410564514244e-09, + "logits/chosen": 0.7580141425132751, + "logits/rejected": 0.6896554231643677, + "logps/chosen": -147.36582946777344, + "logps/ref_chosen": -51.41777801513672, + "logps/ref_rejected": -77.27879333496094, + "logps/rejected": -221.30088806152344, + "loss": 1.0936, + "margin_dpo/margin_mean": 48.074039459228516, + "margin_dpo/margin_std": 71.46519470214844, + "step": 640 + }, + { + "KL/chosen_KL_mean": -97.96511840820312, + "KL/mean": -120.85202026367188, + "KL/rejected_KL_mean": -143.73892211914062, + "KL/std": 63.625396728515625, + "epoch": 0.9690098261526833, + "fcm_dpo/beta": 0.0091469157487154, + "fcm_dpo/delta": -0.018776293843984604, + "fcm_dpo/margin": 45.7738037109375, + "fcm_dpo/q_t": 0.40367794036865234, + "grad_norm": 12.815844535827637, + "learning_rate": 1.5403838846864692e-09, + "logits/chosen": 0.6566144227981567, + "logits/rejected": 0.634056568145752, + "logps/chosen": -169.019775390625, + "logps/ref_chosen": -71.0546646118164, + "logps/ref_rejected": -82.2440185546875, + "logps/rejected": -225.98294067382812, + "loss": 1.0822, + "margin_dpo/margin_mean": 45.7738037109375, + "margin_dpo/margin_std": 60.801631927490234, + "step": 641 + }, + { + "KL/chosen_KL_mean": -105.2142562866211, + "KL/mean": -123.3451156616211, + "KL/rejected_KL_mean": -141.47598266601562, + "KL/std": 61.16352844238281, + "epoch": 0.9705215419501134, + "fcm_dpo/beta": 0.009140715934336185, + "fcm_dpo/delta": -0.03747226670384407, + "fcm_dpo/margin": 36.2617073059082, + "fcm_dpo/q_t": 0.42575782537460327, + "grad_norm": 15.639561653137207, + "learning_rate": 1.3973071544233218e-09, + "logits/chosen": 0.6038320064544678, + "logits/rejected": 0.6211628913879395, + "logps/chosen": -174.14352416992188, + "logps/ref_chosen": -68.92927551269531, + "logps/ref_rejected": -70.85682678222656, + "logps/rejected": -212.3328094482422, + "loss": 1.1902, + "margin_dpo/margin_mean": 36.2617073059082, + "margin_dpo/margin_std": 72.66554260253906, + "step": 642 + }, + { + "KL/chosen_KL_mean": -97.0354995727539, + "KL/mean": -121.27033996582031, + "KL/rejected_KL_mean": -145.50515747070312, + "KL/std": 64.86758422851562, + "epoch": 0.9720332577475435, + "fcm_dpo/beta": 0.009133918210864067, + "fcm_dpo/delta": -0.042914219200611115, + "fcm_dpo/margin": 48.469661712646484, + "fcm_dpo/q_t": 0.4012041985988617, + "grad_norm": 19.11461639404297, + "learning_rate": 1.261184375888541e-09, + "logits/chosen": 0.5985475778579712, + "logits/rejected": 0.5056630373001099, + "logps/chosen": -162.34454345703125, + "logps/ref_chosen": -65.30903625488281, + "logps/ref_rejected": -83.61613464355469, + "logps/rejected": -229.12130737304688, + "loss": 1.1001, + "margin_dpo/margin_mean": 48.46965789794922, + "margin_dpo/margin_std": 74.66355895996094, + "step": 643 + }, + { + "KL/chosen_KL_mean": -87.36654663085938, + "KL/mean": -107.00230407714844, + "KL/rejected_KL_mean": -126.6380615234375, + "KL/std": 62.64691925048828, + "epoch": 0.9735449735449735, + "fcm_dpo/beta": 0.00914299488067627, + "fcm_dpo/delta": 0.04101964831352234, + "fcm_dpo/margin": 39.27149963378906, + "fcm_dpo/q_t": 0.4191191792488098, + "grad_norm": 11.243471145629883, + "learning_rate": 1.1320193567288527e-09, + "logits/chosen": 0.7115193605422974, + "logits/rejected": 0.6787524223327637, + "logps/chosen": -138.36915588378906, + "logps/ref_chosen": -51.002601623535156, + "logps/ref_rejected": -64.46372985839844, + "logps/rejected": -191.10179138183594, + "loss": 1.1757, + "margin_dpo/margin_mean": 39.27149963378906, + "margin_dpo/margin_std": 76.66194152832031, + "step": 644 + }, + { + "KL/chosen_KL_mean": -96.58194732666016, + "KL/mean": -118.76667022705078, + "KL/rejected_KL_mean": -140.95140075683594, + "KL/std": 65.33236694335938, + "epoch": 0.9750566893424036, + "fcm_dpo/beta": 0.009137827903032303, + "fcm_dpo/delta": -0.005471386015415192, + "fcm_dpo/margin": 44.36945724487305, + "fcm_dpo/q_t": 0.40765881538391113, + "grad_norm": 13.416179656982422, + "learning_rate": 1.0098157099674987e-09, + "logits/chosen": 0.6126999855041504, + "logits/rejected": 0.5936166048049927, + "logps/chosen": -157.54534912109375, + "logps/ref_chosen": -60.963409423828125, + "logps/ref_rejected": -69.73353576660156, + "logps/rejected": -210.6849365234375, + "loss": 1.1035, + "margin_dpo/margin_mean": 44.36945724487305, + "margin_dpo/margin_std": 65.06978607177734, + "step": 645 + }, + { + "KL/chosen_KL_mean": -106.65433502197266, + "KL/mean": -127.356201171875, + "KL/rejected_KL_mean": -148.05807495117188, + "KL/std": 72.31283569335938, + "epoch": 0.9765684051398337, + "fcm_dpo/beta": 0.009142119437456131, + "fcm_dpo/delta": 0.021564001217484474, + "fcm_dpo/margin": 41.403751373291016, + "fcm_dpo/q_t": 0.41531550884246826, + "grad_norm": 13.274517059326172, + "learning_rate": 8.945768539031783e-10, + "logits/chosen": 0.6842719316482544, + "logits/rejected": 0.6269073486328125, + "logps/chosen": -168.94439697265625, + "logps/ref_chosen": -62.290069580078125, + "logps/ref_rejected": -85.54812622070312, + "logps/rejected": -233.606201171875, + "loss": 1.1521, + "margin_dpo/margin_mean": 41.40375518798828, + "margin_dpo/margin_std": 75.03099060058594, + "step": 646 + }, + { + "KL/chosen_KL_mean": -102.35749816894531, + "KL/mean": -134.4249725341797, + "KL/rejected_KL_mean": -166.492431640625, + "KL/std": 71.0962905883789, + "epoch": 0.9780801209372638, + "fcm_dpo/beta": 0.009119834750890732, + "fcm_dpo/delta": -0.18595722317695618, + "fcm_dpo/margin": 64.13494110107422, + "fcm_dpo/q_t": 0.3698424994945526, + "grad_norm": 13.934874534606934, + "learning_rate": 7.863060120144316e-10, + "logits/chosen": 0.6742551326751709, + "logits/rejected": 0.574491024017334, + "logps/chosen": -169.87338256835938, + "logps/ref_chosen": -67.515869140625, + "logps/ref_rejected": -101.50871276855469, + "logps/rejected": -268.00115966796875, + "loss": 0.978, + "margin_dpo/margin_mean": 64.13493347167969, + "margin_dpo/margin_std": 70.48902130126953, + "step": 647 + }, + { + "KL/chosen_KL_mean": -103.15081024169922, + "KL/mean": -126.91096496582031, + "KL/rejected_KL_mean": -150.67112731933594, + "KL/std": 62.503177642822266, + "epoch": 0.9795918367346939, + "fcm_dpo/beta": 0.009101306088268757, + "fcm_dpo/delta": -0.032682403922080994, + "fcm_dpo/margin": 47.52030944824219, + "fcm_dpo/q_t": 0.4024064242839813, + "grad_norm": 12.313305854797363, + "learning_rate": 6.850062128694045e-10, + "logits/chosen": 0.6130772829055786, + "logits/rejected": 0.5454744100570679, + "logps/chosen": -167.7467498779297, + "logps/ref_chosen": -64.59593963623047, + "logps/ref_rejected": -83.384033203125, + "logps/rejected": -234.05516052246094, + "loss": 1.1121, + "margin_dpo/margin_mean": 47.52030944824219, + "margin_dpo/margin_std": 75.00138854980469, + "step": 648 + }, + { + "KL/chosen_KL_mean": -96.1046142578125, + "KL/mean": -120.20222473144531, + "KL/rejected_KL_mean": -144.29983520507812, + "KL/std": 64.39717102050781, + "epoch": 0.981103552532124, + "fcm_dpo/beta": 0.009094065986573696, + "fcm_dpo/delta": -0.03851716220378876, + "fcm_dpo/margin": 48.19522476196289, + "fcm_dpo/q_t": 0.40230292081832886, + "grad_norm": 19.866220474243164, + "learning_rate": 5.906802900412788e-10, + "logits/chosen": 0.6978000402450562, + "logits/rejected": 0.634183406829834, + "logps/chosen": -145.4142608642578, + "logps/ref_chosen": -49.30964660644531, + "logps/ref_rejected": -73.73710632324219, + "logps/rejected": -218.0369415283203, + "loss": 1.1055, + "margin_dpo/margin_mean": 48.19522476196289, + "margin_dpo/margin_std": 75.21109008789062, + "step": 649 + }, + { + "KL/chosen_KL_mean": -91.43130493164062, + "KL/mean": -115.72032165527344, + "KL/rejected_KL_mean": -140.00933837890625, + "KL/std": 65.60842895507812, + "epoch": 0.982615268329554, + "fcm_dpo/beta": 0.009089424274861813, + "fcm_dpo/delta": -0.04174076020717621, + "fcm_dpo/margin": 48.57804870605469, + "fcm_dpo/q_t": 0.40230119228363037, + "grad_norm": 13.709752082824707, + "learning_rate": 5.033308820289184e-10, + "logits/chosen": 0.7586034536361694, + "logits/rejected": 0.6871707439422607, + "logps/chosen": -146.49456787109375, + "logps/ref_chosen": -55.06325912475586, + "logps/ref_rejected": -77.39610290527344, + "logps/rejected": -217.40545654296875, + "loss": 1.1094, + "margin_dpo/margin_mean": 48.57804870605469, + "margin_dpo/margin_std": 77.66340637207031, + "step": 650 + }, + { + "KL/chosen_KL_mean": -91.06973266601562, + "KL/mean": -112.2431640625, + "KL/rejected_KL_mean": -133.41659545898438, + "KL/std": 65.72015380859375, + "epoch": 0.9841269841269841, + "fcm_dpo/beta": 0.009093240834772587, + "fcm_dpo/delta": 0.014969753101468086, + "fcm_dpo/margin": 42.34687042236328, + "fcm_dpo/q_t": 0.4142518937587738, + "grad_norm": 14.123266220092773, + "learning_rate": 4.2296043218295606e-10, + "logits/chosen": 0.7175877094268799, + "logits/rejected": 0.6380475163459778, + "logps/chosen": -145.1348876953125, + "logps/ref_chosen": -54.065162658691406, + "logps/ref_rejected": -77.79080200195312, + "logps/rejected": -211.2073974609375, + "loss": 1.1304, + "margin_dpo/margin_mean": 42.34687042236328, + "margin_dpo/margin_std": 68.82353210449219, + "step": 651 + }, + { + "KL/chosen_KL_mean": -107.73420715332031, + "KL/mean": -128.74530029296875, + "KL/rejected_KL_mean": -149.75640869140625, + "KL/std": 67.21748352050781, + "epoch": 0.9856386999244142, + "fcm_dpo/beta": 0.009090578183531761, + "fcm_dpo/delta": 0.01804136112332344, + "fcm_dpo/margin": 42.022193908691406, + "fcm_dpo/q_t": 0.4153905510902405, + "grad_norm": 13.827299118041992, + "learning_rate": 3.4957118863768176e-10, + "logits/chosen": 0.6449884176254272, + "logits/rejected": 0.5916697382926941, + "logps/chosen": -171.37451171875, + "logps/ref_chosen": -63.64030456542969, + "logps/ref_rejected": -78.86882019042969, + "logps/rejected": -228.62522888183594, + "loss": 1.148, + "margin_dpo/margin_mean": 42.022193908691406, + "margin_dpo/margin_std": 74.42575073242188, + "step": 652 + }, + { + "KL/chosen_KL_mean": -95.06451416015625, + "KL/mean": -120.48294067382812, + "KL/rejected_KL_mean": -145.90139770507812, + "KL/std": 65.2015609741211, + "epoch": 0.9871504157218443, + "fcm_dpo/beta": 0.009088236838579178, + "fcm_dpo/delta": -0.06230110302567482, + "fcm_dpo/margin": 50.83686828613281, + "fcm_dpo/q_t": 0.3964662551879883, + "grad_norm": 14.0012788772583, + "learning_rate": 2.831652042480093e-10, + "logits/chosen": 0.6466582417488098, + "logits/rejected": 0.6001366376876831, + "logps/chosen": -156.73287963867188, + "logps/ref_chosen": -61.668373107910156, + "logps/ref_rejected": -73.83012390136719, + "logps/rejected": -219.73150634765625, + "loss": 1.08, + "margin_dpo/margin_mean": 50.83686447143555, + "margin_dpo/margin_std": 73.28072357177734, + "step": 653 + }, + { + "KL/chosen_KL_mean": -99.96598052978516, + "KL/mean": -120.00917053222656, + "KL/rejected_KL_mean": -140.05235290527344, + "KL/std": 72.0395278930664, + "epoch": 0.9886621315192744, + "fcm_dpo/beta": 0.009068830870091915, + "fcm_dpo/delta": -0.08078902959823608, + "fcm_dpo/margin": 40.086368560791016, + "fcm_dpo/q_t": 0.4198213219642639, + "grad_norm": 14.859728813171387, + "learning_rate": 2.2374433653205016e-10, + "logits/chosen": 0.6209405660629272, + "logits/rejected": 0.5149625539779663, + "logps/chosen": -157.53424072265625, + "logps/ref_chosen": -57.568267822265625, + "logps/ref_rejected": -87.74789428710938, + "logps/rejected": -227.8002471923828, + "loss": 1.1533, + "margin_dpo/margin_mean": 40.086368560791016, + "margin_dpo/margin_std": 67.63626098632812, + "step": 654 + }, + { + "KL/chosen_KL_mean": -79.56375122070312, + "KL/mean": -110.12518310546875, + "KL/rejected_KL_mean": -140.68661499023438, + "KL/std": 65.32495880126953, + "epoch": 0.9901738473167044, + "fcm_dpo/beta": 0.009040175005793571, + "fcm_dpo/delta": -0.1536903828382492, + "fcm_dpo/margin": 61.12286376953125, + "fcm_dpo/q_t": 0.3741888999938965, + "grad_norm": 11.832871437072754, + "learning_rate": 1.7131024761923852e-10, + "logits/chosen": 0.6340414881706238, + "logits/rejected": 0.5391018986701965, + "logps/chosen": -131.71090698242188, + "logps/ref_chosen": -52.14714813232422, + "logps/ref_rejected": -80.85014343261719, + "logps/rejected": -221.5367431640625, + "loss": 0.9823, + "margin_dpo/margin_mean": 61.12286376953125, + "margin_dpo/margin_std": 59.80945587158203, + "step": 655 + }, + { + "KL/chosen_KL_mean": -100.05067443847656, + "KL/mean": -125.89352416992188, + "KL/rejected_KL_mean": -151.73635864257812, + "KL/std": 67.03109741210938, + "epoch": 0.9916855631141346, + "fcm_dpo/beta": 0.009031126275658607, + "fcm_dpo/delta": -0.06710033118724823, + "fcm_dpo/margin": 51.685672760009766, + "fcm_dpo/q_t": 0.39476704597473145, + "grad_norm": 11.480228424072266, + "learning_rate": 1.2586440420372934e-10, + "logits/chosen": 0.5607402324676514, + "logits/rejected": 0.5093624591827393, + "logps/chosen": -173.30740356445312, + "logps/ref_chosen": -73.25672912597656, + "logps/ref_rejected": -85.35127258300781, + "logps/rejected": -237.08763122558594, + "loss": 1.0637, + "margin_dpo/margin_mean": 51.685672760009766, + "margin_dpo/margin_std": 69.52171325683594, + "step": 656 + }, + { + "KL/chosen_KL_mean": -91.12415313720703, + "KL/mean": -120.77764892578125, + "KL/rejected_KL_mean": -150.43115234375, + "KL/std": 73.58059692382812, + "epoch": 0.9931972789115646, + "fcm_dpo/beta": 0.00901100691407919, + "fcm_dpo/delta": -0.13513237237930298, + "fcm_dpo/margin": 59.3069953918457, + "fcm_dpo/q_t": 0.38032254576683044, + "grad_norm": 12.082924842834473, + "learning_rate": 8.740807750345913e-11, + "logits/chosen": 0.7595738768577576, + "logits/rejected": 0.6624658107757568, + "logps/chosen": -140.84754943847656, + "logps/ref_chosen": -49.72339630126953, + "logps/ref_rejected": -75.1568603515625, + "logps/rejected": -225.5880126953125, + "loss": 1.0339, + "margin_dpo/margin_mean": 59.3069953918457, + "margin_dpo/margin_std": 77.48700714111328, + "step": 657 + }, + { + "KL/chosen_KL_mean": -98.95183563232422, + "KL/mean": -121.21888732910156, + "KL/rejected_KL_mean": -143.48593139648438, + "KL/std": 72.20649719238281, + "epoch": 0.9947089947089947, + "fcm_dpo/beta": 0.00900908000767231, + "fcm_dpo/delta": -0.0012710578739643097, + "fcm_dpo/margin": 44.53409957885742, + "fcm_dpo/q_t": 0.4118792414665222, + "grad_norm": 13.051859855651855, + "learning_rate": 5.594234322453539e-11, + "logits/chosen": 0.6763231754302979, + "logits/rejected": 0.630454421043396, + "logps/chosen": -161.9981689453125, + "logps/ref_chosen": -63.04634094238281, + "logps/ref_rejected": -83.44963073730469, + "logps/rejected": -226.93557739257812, + "loss": 1.1579, + "margin_dpo/margin_mean": 44.53410339355469, + "margin_dpo/margin_std": 83.93367004394531, + "step": 658 + }, + { + "KL/chosen_KL_mean": -99.3863754272461, + "KL/mean": -119.38518524169922, + "KL/rejected_KL_mean": -139.3839874267578, + "KL/std": 66.35957336425781, + "epoch": 0.9962207105064248, + "fcm_dpo/beta": 0.009003736078739166, + "fcm_dpo/delta": 0.03995537385344505, + "fcm_dpo/margin": 39.997615814208984, + "fcm_dpo/q_t": 0.4187491536140442, + "grad_norm": 14.75331974029541, + "learning_rate": 3.146808153123293e-11, + "logits/chosen": 0.7249075770378113, + "logits/rejected": 0.6584802865982056, + "logps/chosen": -154.46658325195312, + "logps/ref_chosen": -55.0802001953125, + "logps/ref_rejected": -71.91049194335938, + "logps/rejected": -211.29449462890625, + "loss": 1.163, + "margin_dpo/margin_mean": 39.99761199951172, + "margin_dpo/margin_std": 72.67333984375, + "step": 659 + }, + { + "KL/chosen_KL_mean": -98.46337890625, + "KL/mean": -126.13018798828125, + "KL/rejected_KL_mean": -153.79698181152344, + "KL/std": 67.75768280029297, + "epoch": 0.9977324263038548, + "fcm_dpo/beta": 0.008999217301607132, + "fcm_dpo/delta": -0.09844080358743668, + "fcm_dpo/margin": 55.333614349365234, + "fcm_dpo/q_t": 0.38702112436294556, + "grad_norm": 13.27164077758789, + "learning_rate": 1.3985977021235829e-11, + "logits/chosen": 0.7530646324157715, + "logits/rejected": 0.6746411323547363, + "logps/chosen": -152.98928833007812, + "logps/ref_chosen": -54.525917053222656, + "logps/ref_rejected": -81.23604583740234, + "logps/rejected": -235.03302001953125, + "loss": 1.0296, + "margin_dpo/margin_mean": 55.33361053466797, + "margin_dpo/margin_std": 65.84952545166016, + "step": 660 + }, + { + "KL/chosen_KL_mean": -110.1593017578125, + "KL/mean": -130.14324951171875, + "KL/rejected_KL_mean": -150.127197265625, + "KL/std": 70.98070526123047, + "epoch": 0.999244142101285, + "fcm_dpo/beta": 0.009001237340271473, + "fcm_dpo/delta": 0.04036538675427437, + "fcm_dpo/margin": 39.9678840637207, + "fcm_dpo/q_t": 0.4188694655895233, + "grad_norm": 13.019124984741211, + "learning_rate": 3.4965187065971735e-12, + "logits/chosen": 0.6166641712188721, + "logits/rejected": 0.5315142869949341, + "logps/chosen": -170.53195190429688, + "logps/ref_chosen": -60.37263870239258, + "logps/ref_rejected": -77.42874145507812, + "logps/rejected": -227.55593872070312, + "loss": 1.1713, + "margin_dpo/margin_mean": 39.96788787841797, + "margin_dpo/margin_std": 77.18132019042969, + "step": 661 + }, + { + "epoch": 0.999244142101285, + "step": 661, + "total_flos": 0.0, + "train_loss": 1.1039659273606386, + "train_runtime": 1648.6122, + "train_samples_per_second": 25.68, + "train_steps_per_second": 0.401 + } + ], + "logging_steps": 1, + "max_steps": 661, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}