From 5228db97ccf346769c466f7169ffea7c67f1a298 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 22 Jul 2026 04:24:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-p3-ronpo-alpha050_anchor0035_lr7p5e8-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4903 ++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 13 files changed, 5250 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b1eb11c --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-p3-ronpo-alpha050_anchor0035_lr7p5e8-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ronpo_full_expect_sweep_alpha050_anchor0035_lr7p5e8 +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/eval/p3_ronpo_seed42_sweep/candidates/alpha050_anchor0035_lr7p5e8` +- Uploaded at UTC: 2026-07-14T01:26:11Z +- Run status metadata: `not found` + +Seed-42 P3 candidate selected only on non-sealed validation after strict S3 pass. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..afc867e --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.4220078429910872, + "train_runtime": 7042.2326, + "train_samples": 50340, + "train_samples_per_second": 2.045, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..0ea06f9 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.035 +preference_sft_weight: 0.0035 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 7.5e-08 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/eval/p3_ronpo_seed42_sweep/candidates/alpha050_anchor0035_lr7p5e8 +run_name: aaai27-p3-ronpo-alpha050_anchor0035_lr7p5e8-s42 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..76651de --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97960bd9df40215c4621be41e9075dfc3fc781ae3e4f1a0139e005c5573c5d24 +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..afc867e --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.4220078429910872, + "train_runtime": 7042.2326, + "train_samples": 50340, + "train_samples_per_second": 2.045, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..6f2b482 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.2888891100883484, + "drift/rejected_abs": 0.17491090297698975, + "epoch": 0.0015891934843067143, + "grad_norm": 0.90625, + "learning_rate": 3.3333333333333334e-09, + "logits/chosen": -2.22796893119812, + "logits/rejected": -2.1537139415740967, + "logps/chosen": -0.25039365887641907, + "logps/rejected": -0.2549845278263092, + "loss": 0.2590796649456024, + "loss/core": 0.24525848031044006, + "loss/preference_sft": 0.2586572766304016, + "loss/reference_anchor": 0.3690255582332611, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8888912200927734, + "rewards/margins": 1.140271782875061, + "rewards/rejected": 1.7486193180084229, + "ronpo/logit": 0.11402717977762222, + "ronpo/residual": 0.12274861335754395, + "ronpo/residual_abs": 0.22723272442817688, + "ronpo/target": -0.008721430785953999, + "ronpo/target_abs": 0.06909500062465668, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.25118276476860046, + "drift/rejected_abs": 0.1736268401145935, + "epoch": 0.0031783869686134287, + "grad_norm": 5.21875, + "learning_rate": 7.5e-09, + "logits/chosen": -1.9810645580291748, + "logits/rejected": -1.9216926097869873, + "logps/chosen": -0.2643340528011322, + "logps/rejected": -0.2714827060699463, + "loss": 0.15867051482200623, + "loss/core": 0.14627718925476074, + "loss/preference_sft": 0.2700865864753723, + "loss/reference_anchor": 0.3270857632160187, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.511749267578125, + "rewards/margins": 0.7762027978897095, + "rewards/rejected": 1.735546350479126, + "ronpo/logit": 0.0776202604174614, + "ronpo/residual": 0.0606955885887146, + "ronpo/residual_abs": 0.17922377586364746, + "ronpo/target": 0.016924677416682243, + "ronpo/target_abs": 0.08231900632381439, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.14721375703811646, + "drift/rejected_abs": 0.0799868255853653, + "epoch": 0.004767580452920143, + "grad_norm": 7.5, + "learning_rate": 1.1666666666666667e-08, + "logits/chosen": -2.291767120361328, + "logits/rejected": -2.2457759380340576, + "logps/chosen": -0.2763332426548004, + "logps/rejected": -0.28252172470092773, + "loss": 0.049369972199201584, + "loss/core": 0.04628817364573479, + "loss/preference_sft": 0.2778027057647705, + "loss/reference_anchor": 0.060271114110946655, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4715213775634766, + "rewards/margins": 0.6733793020248413, + "rewards/rejected": 0.7981420755386353, + "ronpo/logit": 0.06733792275190353, + "ronpo/residual": 0.09835626929998398, + "ronpo/residual_abs": 0.1334967315196991, + "ronpo/target": -0.031018350273370743, + "ronpo/target_abs": 0.076286181807518, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.14123335480690002, + "drift/rejected_abs": 0.0699237510561943, + "epoch": 0.006356773937226857, + "grad_norm": 1.1171875, + "learning_rate": 1.5833333333333333e-08, + "logits/chosen": -2.2912278175354004, + "logits/rejected": -2.2170040607452393, + "logps/chosen": -0.27552682161331177, + "logps/rejected": -0.27909940481185913, + "loss": 0.02950289286673069, + "loss/core": 0.02670697309076786, + "loss/preference_sft": 0.27772125601768494, + "loss/reference_anchor": 0.05211133509874344, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.4118921756744385, + "rewards/margins": 0.7152848839759827, + "rewards/rejected": 0.6966072916984558, + "ronpo/logit": 0.07152849435806274, + "ronpo/residual": 0.06994546204805374, + "ronpo/residual_abs": 0.10728304088115692, + "ronpo/target": 0.001583026722073555, + "ronpo/target_abs": 0.0675927996635437, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.23433640599250793, + "drift/rejected_abs": 0.07729706168174744, + "epoch": 0.007945967421533572, + "grad_norm": 1.125, + "learning_rate": 2e-08, + "logits/chosen": -2.353553295135498, + "logits/rejected": -2.392822504043579, + "logps/chosen": -0.23983506858348846, + "logps/rejected": -0.24817518889904022, + "loss": 0.30295810103416443, + "loss/core": 0.29502326250076294, + "loss/preference_sft": 0.2504972517490387, + "loss/reference_anchor": 0.20166020095348358, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.342874050140381, + "rewards/margins": 1.5709974765777588, + "rewards/rejected": 0.7718764543533325, + "ronpo/logit": 0.15709975361824036, + "ronpo/residual": 0.1602698713541031, + "ronpo/residual_abs": 0.20184986293315887, + "ronpo/target": -0.0031701307743787766, + "ronpo/target_abs": 0.07585608959197998, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.20710475742816925, + "drift/rejected_abs": 0.0870819017291069, + "epoch": 0.009535160905840286, + "grad_norm": 316.0, + "learning_rate": 2.4166666666666668e-08, + "logits/chosen": -2.2282848358154297, + "logits/rejected": -2.124019145965576, + "logps/chosen": -0.28481242060661316, + "logps/rejected": -0.28323936462402344, + "loss": 0.1839178502559662, + "loss/core": 0.17509128153324127, + "loss/preference_sft": 0.2906222939491272, + "loss/reference_anchor": 0.22312648594379425, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0701568126678467, + "rewards/margins": 1.1993378400802612, + "rewards/rejected": 0.870819091796875, + "ronpo/logit": 0.11993379890918732, + "ronpo/residual": 0.10416283458471298, + "ronpo/residual_abs": 0.17567749321460724, + "ronpo/target": 0.015770956873893738, + "ronpo/target_abs": 0.0844925194978714, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.20917043089866638, + "drift/rejected_abs": 0.13052448630332947, + "epoch": 0.011124354390147001, + "grad_norm": 2.3125, + "learning_rate": 2.8333333333333332e-08, + "logits/chosen": -2.122420310974121, + "logits/rejected": -2.0597376823425293, + "logps/chosen": -0.2706679403781891, + "logps/rejected": -0.28155890107154846, + "loss": 0.04921208694577217, + "loss/core": 0.04241333529353142, + "loss/preference_sft": 0.2808118462562561, + "loss/reference_anchor": 0.16616889834403992, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.091449737548828, + "rewards/margins": 0.7882283926010132, + "rewards/rejected": 1.303221344947815, + "ronpo/logit": 0.07882285118103027, + "ronpo/residual": 0.06731168925762177, + "ronpo/residual_abs": 0.12287186086177826, + "ronpo/target": 0.011511145159602165, + "ronpo/target_abs": 0.07380068302154541, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.13368353247642517, + "drift/rejected_abs": 0.10889682918787003, + "epoch": 0.012713547874453715, + "grad_norm": 430.0, + "learning_rate": 3.25e-08, + "logits/chosen": -2.080500364303589, + "logits/rejected": -2.039989471435547, + "logps/chosen": -0.31930357217788696, + "logps/rejected": -0.32469144463539124, + "loss": 0.22682690620422363, + "loss/core": 0.220375657081604, + "loss/preference_sft": 0.32273247838020325, + "loss/reference_anchor": 0.15204796195030212, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.335686445236206, + "rewards/margins": 0.24776522815227509, + "rewards/rejected": 1.0879215002059937, + "ronpo/logit": 0.024776523932814598, + "ronpo/residual": 0.01710623875260353, + "ronpo/residual_abs": 0.15854685008525848, + "ronpo/target": 0.007670276798307896, + "ronpo/target_abs": 0.06935645639896393, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.19959691166877747, + "drift/rejected_abs": 0.10330881178379059, + "epoch": 0.014302741358760428, + "grad_norm": 3.90625, + "learning_rate": 3.6666666666666664e-08, + "logits/chosen": -2.1476857662200928, + "logits/rejected": -2.0756335258483887, + "logps/chosen": -0.30488723516464233, + "logps/rejected": -0.29676884412765503, + "loss": 0.08849599957466125, + "loss/core": 0.08234541118144989, + "loss/preference_sft": 0.3082302510738373, + "loss/reference_anchor": 0.14490821957588196, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9952192306518555, + "rewards/margins": 0.9640136957168579, + "rewards/rejected": 1.0312055349349976, + "ronpo/logit": 0.09640135616064072, + "ronpo/residual": 0.09527552872896194, + "ronpo/residual_abs": 0.14935743808746338, + "ronpo/target": 0.0011258311569690704, + "ronpo/target_abs": 0.0794779434800148, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.20762768387794495, + "drift/rejected_abs": 0.17166680097579956, + "epoch": 0.015891934843067144, + "grad_norm": 8.5625, + "learning_rate": 4.083333333333333e-08, + "logits/chosen": -2.2317585945129395, + "logits/rejected": -2.281120538711548, + "logps/chosen": -0.38824811577796936, + "logps/rejected": -0.37904253602027893, + "loss": 0.48683279752731323, + "loss/core": 0.4693141579627991, + "loss/preference_sft": 0.3856825530529022, + "loss/reference_anchor": 0.4619651436805725, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.075303554534912, + "rewards/margins": 1.4245798587799072, + "rewards/rejected": 0.6507236361503601, + "ronpo/logit": 0.1424579918384552, + "ronpo/residual": 0.1403721421957016, + "ronpo/residual_abs": 0.23278239369392395, + "ronpo/target": 0.002085856394842267, + "ronpo/target_abs": 0.07827635854482651, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.2422821968793869, + "drift/rejected_abs": 0.11313442885875702, + "epoch": 0.01748112832737386, + "grad_norm": 1.4140625, + "learning_rate": 4.5e-08, + "logits/chosen": -2.13415265083313, + "logits/rejected": -2.1069016456604004, + "logps/chosen": -0.3025519549846649, + "logps/rejected": -0.29403483867645264, + "loss": 0.2851434350013733, + "loss/core": 0.27573922276496887, + "loss/preference_sft": 0.29142332077026367, + "loss/reference_anchor": 0.2395510971546173, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.42053484916687, + "rewards/margins": 1.2919930219650269, + "rewards/rejected": 1.1285418272018433, + "ronpo/logit": 0.1291993260383606, + "ronpo/residual": 0.12877634167671204, + "ronpo/residual_abs": 0.18722160160541534, + "ronpo/target": 0.00042299646884202957, + "ronpo/target_abs": 0.0751943588256836, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.2859105169773102, + "drift/rejected_abs": 0.07585777342319489, + "epoch": 0.01907032181168057, + "grad_norm": 6.75, + "learning_rate": 4.9166666666666664e-08, + "logits/chosen": -2.022763252258301, + "logits/rejected": -2.0409018993377686, + "logps/chosen": -0.2585347890853882, + "logps/rejected": -0.26091212034225464, + "loss": 0.6952535510063171, + "loss/core": 0.6805825233459473, + "loss/preference_sft": 0.2590915560722351, + "loss/reference_anchor": 0.3932636082172394, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.8586783409118652, + "rewards/margins": 2.1142590045928955, + "rewards/rejected": 0.7444195747375488, + "ronpo/logit": 0.21142590045928955, + "ronpo/residual": 0.22805149853229523, + "ronpo/residual_abs": 0.26630499958992004, + "ronpo/target": -0.01662559062242508, + "ronpo/target_abs": 0.07312311232089996, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.23467068374156952, + "drift/rejected_abs": 0.08460647612810135, + "epoch": 0.020659515295987287, + "grad_norm": 156.0, + "learning_rate": 5.3333333333333334e-08, + "logits/chosen": -2.391279458999634, + "logits/rejected": -2.461313247680664, + "logps/chosen": -0.2505510151386261, + "logps/rejected": -0.24410393834114075, + "loss": 0.23585395514965057, + "loss/core": 0.22895380854606628, + "loss/preference_sft": 0.25251513719558716, + "loss/reference_anchor": 0.1718951165676117, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3467066287994385, + "rewards/margins": 1.5009348392486572, + "rewards/rejected": 0.8457720875740051, + "ronpo/logit": 0.15009348094463348, + "ronpo/residual": 0.12950626015663147, + "ronpo/residual_abs": 0.1809195727109909, + "ronpo/target": 0.02058723010122776, + "ronpo/target_abs": 0.07401974499225616, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.2189807891845703, + "drift/rejected_abs": 0.11199836432933807, + "epoch": 0.022248708780294002, + "grad_norm": 3.640625, + "learning_rate": 5.75e-08, + "logits/chosen": -2.13312029838562, + "logits/rejected": -2.1325230598449707, + "logps/chosen": -0.2632366716861725, + "logps/rejected": -0.26798808574676514, + "loss": 0.07575295865535736, + "loss/core": 0.07079951465129852, + "loss/preference_sft": 0.27200424671173096, + "loss/reference_anchor": 0.11432687193155289, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.18943190574646, + "rewards/margins": 1.069993495941162, + "rewards/rejected": 1.1194385290145874, + "ronpo/logit": 0.10699935257434845, + "ronpo/residual": 0.09644700586795807, + "ronpo/residual_abs": 0.1400878131389618, + "ronpo/target": 0.010552345775067806, + "ronpo/target_abs": 0.07036933302879333, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.2812674641609192, + "drift/rejected_abs": 0.14779499173164368, + "epoch": 0.023837902264600714, + "grad_norm": 6.65625, + "learning_rate": 6.166666666666666e-08, + "logits/chosen": -2.1063144207000732, + "logits/rejected": -2.0400238037109375, + "logps/chosen": -0.2795808017253876, + "logps/rejected": -0.27232033014297485, + "loss": 0.6679071187973022, + "loss/core": 0.6522617340087891, + "loss/preference_sft": 0.27452582120895386, + "loss/reference_anchor": 0.4195597171783447, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8126060962677, + "rewards/margins": 1.365581750869751, + "rewards/rejected": 1.4470244646072388, + "ronpo/logit": 0.1365581750869751, + "ronpo/residual": 0.14009013772010803, + "ronpo/residual_abs": 0.30546045303344727, + "ronpo/target": -0.003531957510858774, + "ronpo/target_abs": 0.0828089714050293, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.32651928067207336, + "drift/rejected_abs": 0.1381281465291977, + "epoch": 0.02542709574890743, + "grad_norm": 41.0, + "learning_rate": 6.583333333333333e-08, + "logits/chosen": -2.0517067909240723, + "logits/rejected": -2.019787311553955, + "logps/chosen": -0.270771861076355, + "logps/rejected": -0.27388402819633484, + "loss": 0.371652215719223, + "loss/core": 0.35672515630722046, + "loss/preference_sft": 0.27029484510421753, + "loss/reference_anchor": 0.3994576334953308, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.265192747116089, + "rewards/margins": 1.8877226114273071, + "rewards/rejected": 1.3774702548980713, + "ronpo/logit": 0.18877224624156952, + "ronpo/residual": 0.18959882855415344, + "ronpo/residual_abs": 0.23960533738136292, + "ronpo/target": -0.0008265804499387741, + "ronpo/target_abs": 0.08044739812612534, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.229955792427063, + "drift/rejected_abs": 0.07845465838909149, + "epoch": 0.027016289233214145, + "grad_norm": 1000.0, + "learning_rate": 6.999999999999999e-08, + "logits/chosen": -2.366101026535034, + "logits/rejected": -2.30869722366333, + "logps/chosen": -0.29234808683395386, + "logps/rejected": -0.26663321256637573, + "loss": 0.579170286655426, + "loss/core": 0.5666659474372864, + "loss/preference_sft": 0.2720109522342682, + "loss/reference_anchor": 0.3300662636756897, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2987306118011475, + "rewards/margins": 1.5194584131240845, + "rewards/rejected": 0.779272198677063, + "ronpo/logit": 0.15194585919380188, + "ronpo/residual": 0.1387050896883011, + "ronpo/residual_abs": 0.20297744870185852, + "ronpo/target": 0.013240769505500793, + "ronpo/target_abs": 0.07829339802265167, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.1896975040435791, + "drift/rejected_abs": 0.16012468934059143, + "epoch": 0.028605482717520857, + "grad_norm": 1.1875, + "learning_rate": 7.416666666666666e-08, + "logits/chosen": -2.2618637084960938, + "logits/rejected": -2.2726664543151855, + "logps/chosen": -0.2831419110298157, + "logps/rejected": -0.2784458100795746, + "loss": 0.26859554648399353, + "loss/core": 0.26028066873550415, + "loss/preference_sft": 0.2823284864425659, + "loss/reference_anchor": 0.20933446288108826, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8969752788543701, + "rewards/margins": 0.29952192306518555, + "rewards/rejected": 1.5974533557891846, + "ronpo/logit": 0.029952192679047585, + "ronpo/residual": 0.023403458297252655, + "ronpo/residual_abs": 0.17928209900856018, + "ronpo/target": 0.006548731122165918, + "ronpo/target_abs": 0.08282124251127243, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.22641654312610626, + "drift/rejected_abs": 0.10155995935201645, + "epoch": 0.030194676201827572, + "grad_norm": 3.8125, + "learning_rate": 7.499548723985053e-08, + "logits/chosen": -2.3383660316467285, + "logits/rejected": -2.2575364112854004, + "logps/chosen": -0.2721652388572693, + "logps/rejected": -0.270842969417572, + "loss": 0.3992326259613037, + "loss/core": 0.39081233739852905, + "loss/preference_sft": 0.2746056616306305, + "loss/reference_anchor": 0.21311792731285095, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2641654014587402, + "rewards/margins": 1.2714884281158447, + "rewards/rejected": 0.9926770925521851, + "ronpo/logit": 0.12714883685112, + "ronpo/residual": 0.14186282455921173, + "ronpo/residual_abs": 0.22912614047527313, + "ronpo/target": -0.014713977463543415, + "ronpo/target_abs": 0.08614644408226013, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.253028929233551, + "drift/rejected_abs": 0.10068502277135849, + "epoch": 0.03178386968613429, + "grad_norm": 10.0, + "learning_rate": 7.497715601321609e-08, + "logits/chosen": -1.9719501733779907, + "logits/rejected": -2.0437729358673096, + "logps/chosen": -0.2982824444770813, + "logps/rejected": -0.28864842653274536, + "loss": 0.24105651676654816, + "loss/core": 0.23307080566883087, + "loss/preference_sft": 0.28916245698928833, + "loss/reference_anchor": 0.19924673438072205, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5282044410705566, + "rewards/margins": 1.5234105587005615, + "rewards/rejected": 1.0047937631607056, + "ronpo/logit": 0.1523410826921463, + "ronpo/residual": 0.1546805202960968, + "ronpo/residual_abs": 0.19531986117362976, + "ronpo/target": -0.0023394643794745207, + "ronpo/target_abs": 0.07932108640670776, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.3908888101577759, + "drift/rejected_abs": 0.12839046120643616, + "epoch": 0.033373063170441, + "grad_norm": 95.0, + "learning_rate": 7.494473116090843e-08, + "logits/chosen": -1.92897629737854, + "logits/rejected": -1.9361374378204346, + "logps/chosen": -0.2795645296573639, + "logps/rejected": -0.28167638182640076, + "loss": 0.6834398508071899, + "loss/core": 0.6608570218086243, + "loss/preference_sft": 0.2787085473537445, + "loss/reference_anchor": 0.617350697517395, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.908715009689331, + "rewards/margins": 2.635611057281494, + "rewards/rejected": 1.273103952407837, + "ronpo/logit": 0.26356109976768494, + "ronpo/residual": 0.27055853605270386, + "ronpo/residual_abs": 0.3101016879081726, + "ronpo/target": -0.00699741393327713, + "ronpo/target_abs": 0.08270318806171417, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.2439744919538498, + "drift/rejected_abs": 0.12347511947154999, + "epoch": 0.03496225665474772, + "grad_norm": 8.0, + "learning_rate": 7.489822487658843e-08, + "logits/chosen": -2.243443250656128, + "logits/rejected": -2.2027456760406494, + "logps/chosen": -0.29370397329330444, + "logps/rejected": -0.28651857376098633, + "loss": 0.12330013513565063, + "loss/core": 0.11514942348003387, + "loss/preference_sft": 0.29073047637939453, + "loss/reference_anchor": 0.20380404591560364, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.438559055328369, + "rewards/margins": 1.2159253358840942, + "rewards/rejected": 1.2226336002349854, + "ronpo/logit": 0.12159254401922226, + "ronpo/residual": 0.12451876699924469, + "ronpo/residual_abs": 0.18701641261577606, + "ronpo/target": -0.002926218556240201, + "ronpo/target_abs": 0.08683927357196808, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.27595922350883484, + "drift/rejected_abs": 0.1378968507051468, + "epoch": 0.03655145013905443, + "grad_norm": 2.1875, + "learning_rate": 7.483765464936766e-08, + "logits/chosen": -1.8842188119888306, + "logits/rejected": -1.7752964496612549, + "logps/chosen": -0.2784324586391449, + "logps/rejected": -0.2820046842098236, + "loss": 0.1508234441280365, + "loss/core": 0.14237406849861145, + "loss/preference_sft": 0.2840380072593689, + "loss/reference_anchor": 0.21300740540027618, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.757049560546875, + "rewards/margins": 1.3863294124603271, + "rewards/rejected": 1.3707202672958374, + "ronpo/logit": 0.13863296806812286, + "ronpo/residual": 0.14248371124267578, + "ronpo/residual_abs": 0.18661606311798096, + "ronpo/target": -0.003850756911560893, + "ronpo/target_abs": 0.07908544689416885, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.25356635451316833, + "drift/rejected_abs": 0.11751619726419449, + "epoch": 0.03814064362336114, + "grad_norm": 1.078125, + "learning_rate": 7.476304325723143e-08, + "logits/chosen": -2.017167568206787, + "logits/rejected": -2.0651519298553467, + "logps/chosen": -0.3016659915447235, + "logps/rejected": -0.2997520864009857, + "loss": 0.2998911142349243, + "loss/core": 0.28765398263931274, + "loss/preference_sft": 0.3130350410938263, + "loss/reference_anchor": 0.31832972168922424, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.534654140472412, + "rewards/margins": 1.3620463609695435, + "rewards/rejected": 1.1726077795028687, + "ronpo/logit": 0.13620463013648987, + "ronpo/residual": 0.13591532409191132, + "ronpo/residual_abs": 0.20360629260540009, + "ronpo/target": 0.0002892996999435127, + "ronpo/target_abs": 0.07245339453220367, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.1270354688167572, + "drift/rejected_abs": 0.07065033167600632, + "epoch": 0.03972983710766786, + "grad_norm": 1.3359375, + "learning_rate": 7.467441875847293e-08, + "logits/chosen": -2.2720069885253906, + "logits/rejected": -2.192866802215576, + "logps/chosen": -0.2744554877281189, + "logps/rejected": -0.28156858682632446, + "loss": 0.04993171617388725, + "loss/core": 0.04652109742164612, + "loss/preference_sft": 0.2717655599117279, + "loss/reference_anchor": 0.07026959955692291, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2697632312774658, + "rewards/margins": 0.5650413036346436, + "rewards/rejected": 0.7047218680381775, + "ronpo/logit": 0.05650413781404495, + "ronpo/residual": 0.06540760397911072, + "ronpo/residual_abs": 0.11058205366134644, + "ronpo/target": -0.008903468027710915, + "ronpo/target_abs": 0.07374493777751923, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.3694007992744446, + "drift/rejected_abs": 0.15201464295387268, + "epoch": 0.04131903059197457, + "grad_norm": 16.875, + "learning_rate": 7.457181448114169e-08, + "logits/chosen": -2.0490753650665283, + "logits/rejected": -1.9781653881072998, + "logps/chosen": -0.280802845954895, + "logps/rejected": -0.30160802602767944, + "loss": 0.4936361312866211, + "loss/core": 0.4678853154182434, + "loss/preference_sft": 0.2995707094669342, + "loss/reference_anchor": 0.7057813405990601, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.6940078735351562, + "rewards/margins": 2.1743321418762207, + "rewards/rejected": 1.5196759700775146, + "ronpo/logit": 0.21743318438529968, + "ronpo/residual": 0.2153353989124298, + "ronpo/residual_abs": 0.2628825604915619, + "ronpo/target": 0.0020978045649826527, + "ronpo/target_abs": 0.06583398580551147, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.22835186123847961, + "drift/rejected_abs": 0.17439039051532745, + "epoch": 0.04290822407628129, + "grad_norm": 90.0, + "learning_rate": 7.445526901051018e-08, + "logits/chosen": -2.157517910003662, + "logits/rejected": -2.1436400413513184, + "logps/chosen": -0.2549034357070923, + "logps/rejected": -0.27489176392555237, + "loss": 0.35979098081588745, + "loss/core": 0.3489346206188202, + "loss/preference_sft": 0.26387128233909607, + "loss/reference_anchor": 0.28379374742507935, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2820842266082764, + "rewards/margins": 0.5401636958122253, + "rewards/rejected": 1.7419202327728271, + "ronpo/logit": 0.054016370326280594, + "ronpo/residual": 0.05434161424636841, + "ronpo/residual_abs": 0.2039295881986618, + "ronpo/target": -0.0003252413007430732, + "ronpo/target_abs": 0.06969951838254929, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.2118876725435257, + "drift/rejected_abs": 0.1539703756570816, + "epoch": 0.044497417560588004, + "grad_norm": 38.75, + "learning_rate": 7.432482617456355e-08, + "logits/chosen": -2.0652270317077637, + "logits/rejected": -2.0345349311828613, + "logps/chosen": -0.28724318742752075, + "logps/rejected": -0.28770583868026733, + "loss": 0.2550918161869049, + "loss/core": 0.2464626580476761, + "loss/preference_sft": 0.28313058614730835, + "loss/reference_anchor": 0.2182338684797287, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.117487668991089, + "rewards/margins": 0.5793629884719849, + "rewards/rejected": 1.538124918937683, + "ronpo/logit": 0.057936299592256546, + "ronpo/residual": 0.04774229973554611, + "ronpo/residual_abs": 0.19685055315494537, + "ronpo/target": 0.010194000788033009, + "ronpo/target_abs": 0.0766717940568924, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.3104633092880249, + "drift/rejected_abs": 0.13752777874469757, + "epoch": 0.04608661104489471, + "grad_norm": 213.0, + "learning_rate": 7.41805350275177e-08, + "logits/chosen": -2.1027872562408447, + "logits/rejected": -2.1394100189208984, + "logps/chosen": -0.2751466631889343, + "logps/rejected": -0.2653554081916809, + "loss": 0.20231513679027557, + "loss/core": 0.18823571503162384, + "loss/preference_sft": 0.2743663191795349, + "loss/reference_anchor": 0.37483280897140503, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.104318380355835, + "rewards/margins": 1.7324186563491821, + "rewards/rejected": 1.3718997240066528, + "ronpo/logit": 0.17324186861515045, + "ronpo/residual": 0.1822124421596527, + "ronpo/residual_abs": 0.214836984872818, + "ronpo/target": -0.008970573544502258, + "ronpo/target_abs": 0.07618303596973419, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.40220266580581665, + "drift/rejected_abs": 0.11431588977575302, + "epoch": 0.04767580452920143, + "grad_norm": 2.265625, + "learning_rate": 7.402244983137196e-08, + "logits/chosen": -2.0275425910949707, + "logits/rejected": -2.13136625289917, + "logps/chosen": -0.29549291729927063, + "logps/rejected": -0.2743282914161682, + "loss": 0.994286835193634, + "loss/core": 0.9691765904426575, + "loss/preference_sft": 0.2976817190647125, + "loss/reference_anchor": 0.6876654624938965, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.022026062011719, + "rewards/margins": 2.8812575340270996, + "rewards/rejected": 1.1407687664031982, + "ronpo/logit": 0.28812575340270996, + "ronpo/residual": 0.2670125663280487, + "ronpo/residual_abs": 0.32439857721328735, + "ronpo/target": 0.021113192662596703, + "ronpo/target_abs": 0.08010004460811615, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.311504065990448, + "drift/rejected_abs": 0.10349141061306, + "epoch": 0.04926499801350814, + "grad_norm": 4.875, + "learning_rate": 7.385063003550334e-08, + "logits/chosen": -2.169419765472412, + "logits/rejected": -2.1742537021636963, + "logps/chosen": -0.2659375071525574, + "logps/rejected": -0.26293236017227173, + "loss": 0.6533135175704956, + "loss/core": 0.6341246366500854, + "loss/preference_sft": 0.2658475339412689, + "loss/reference_anchor": 0.5216662287712097, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1136386394500732, + "rewards/margins": 2.081437826156616, + "rewards/rejected": 1.0322009325027466, + "ronpo/logit": 0.20814375579357147, + "ronpo/residual": 0.20334522426128387, + "ronpo/residual_abs": 0.2493644505739212, + "ronpo/target": 0.004798541776835918, + "ronpo/target_abs": 0.07737194001674652, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.20044144988059998, + "drift/rejected_abs": 0.06772055476903915, + "epoch": 0.05085419149781486, + "grad_norm": 844.0, + "learning_rate": 7.36651402543102e-08, + "logits/chosen": -2.206714153289795, + "logits/rejected": -2.2234230041503906, + "logps/chosen": -0.3195026218891144, + "logps/rejected": -0.27392783761024475, + "loss": 0.5297176241874695, + "loss/core": 0.517520546913147, + "loss/preference_sft": 0.3230019211769104, + "loss/reference_anchor": 0.3161885142326355, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0040805339813232, + "rewards/margins": 1.3281657695770264, + "rewards/rejected": 0.6759150624275208, + "ronpo/logit": 0.13281656801700592, + "ronpo/residual": 0.12692351639270782, + "ronpo/residual_abs": 0.18193206191062927, + "ronpo/target": 0.00589305954053998, + "ronpo/target_abs": 0.0840735211968422, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.1306372582912445, + "drift/rejected_abs": 0.0512150414288044, + "epoch": 0.052443384982121574, + "grad_norm": 4.46875, + "learning_rate": 7.346605024291327e-08, + "logits/chosen": -2.128614902496338, + "logits/rejected": -2.0060853958129883, + "logps/chosen": -0.2849295735359192, + "logps/rejected": -0.2936997413635254, + "loss": 0.11120221763849258, + "loss/core": 0.10780216753482819, + "loss/preference_sft": 0.28842681646347046, + "loss/reference_anchor": 0.06830208003520966, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3055182695388794, + "rewards/margins": 0.7997378706932068, + "rewards/rejected": 0.5057803392410278, + "ronpo/logit": 0.07997378706932068, + "ronpo/residual": 0.07414297759532928, + "ronpo/residual_abs": 0.13741916418075562, + "ronpo/target": 0.005830812267959118, + "ronpo/target_abs": 0.07218202203512192, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.31605249643325806, + "drift/rejected_abs": 0.16812530159950256, + "epoch": 0.05403257846642829, + "grad_norm": 1.0703125, + "learning_rate": 7.325343487092362e-08, + "logits/chosen": -2.0245869159698486, + "logits/rejected": -2.0181970596313477, + "logps/chosen": -0.28985458612442017, + "logps/rejected": -0.2747141420841217, + "loss": 0.4409272074699402, + "loss/core": 0.42512696981430054, + "loss/preference_sft": 0.2895664572715759, + "loss/reference_anchor": 0.42247915267944336, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.159921407699585, + "rewards/margins": 1.4804481267929077, + "rewards/rejected": 1.6794731616973877, + "ronpo/logit": 0.14804483950138092, + "ronpo/residual": 0.14191299676895142, + "ronpo/residual_abs": 0.25526586174964905, + "ronpo/target": 0.006131820380687714, + "ronpo/target_abs": 0.07981385290622711, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.19817472994327545, + "drift/rejected_abs": 0.11151386797428131, + "epoch": 0.055621771950735005, + "grad_norm": 26.375, + "learning_rate": 7.302737409428734e-08, + "logits/chosen": -2.2624623775482178, + "logits/rejected": -2.1980812549591064, + "logps/chosen": -0.29418444633483887, + "logps/rejected": -0.30473509430885315, + "loss": 0.05296384170651436, + "loss/core": 0.0463070385158062, + "loss/preference_sft": 0.2961450517177582, + "loss/reference_anchor": 0.1605798900127411, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9817469120025635, + "rewards/margins": 0.8666085004806519, + "rewards/rejected": 1.1151385307312012, + "ronpo/logit": 0.08666086941957474, + "ronpo/residual": 0.09623171389102936, + "ronpo/residual_abs": 0.13510657846927643, + "ronpo/target": -0.009570842608809471, + "ronpo/target_abs": 0.08157099038362503, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.47317036986351013, + "drift/rejected_abs": 0.22457854449748993, + "epoch": 0.057210965435041714, + "grad_norm": 30.125, + "learning_rate": 7.278795292521733e-08, + "logits/chosen": -2.0715465545654297, + "logits/rejected": -2.0496222972869873, + "logps/chosen": -0.28974977135658264, + "logps/rejected": -0.28343138098716736, + "loss": 0.9135662913322449, + "loss/core": 0.8731704950332642, + "loss/preference_sft": 0.2855917811393738, + "loss/reference_anchor": 1.1256051063537598, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.731703758239746, + "rewards/margins": 2.486184597015381, + "rewards/rejected": 2.2455201148986816, + "ronpo/logit": 0.24861843883991241, + "ronpo/residual": 0.26165086030960083, + "ronpo/residual_abs": 0.29524511098861694, + "ronpo/target": -0.013032438233494759, + "ronpo/target_abs": 0.0726519376039505, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.12218070030212402, + "drift/rejected_abs": 0.06443633139133453, + "epoch": 0.05880015891934843, + "grad_norm": 4.59375, + "learning_rate": 7.253526140022365e-08, + "logits/chosen": -2.228654384613037, + "logits/rejected": -2.33091139793396, + "logps/chosen": -0.29517388343811035, + "logps/rejected": -0.2922295331954956, + "loss": 0.027977192774415016, + "loss/core": 0.025672132149338722, + "loss/preference_sft": 0.2922584116458893, + "loss/reference_anchor": 0.03663304075598717, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.2214815616607666, + "rewards/margins": 0.5821638107299805, + "rewards/rejected": 0.6393176317214966, + "ronpo/logit": 0.05821638181805611, + "ronpo/residual": 0.06779725104570389, + "ronpo/residual_abs": 0.1186339408159256, + "ronpo/target": -0.009580868296325207, + "ronpo/target_abs": 0.08872409909963608, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.12466661632061005, + "drift/rejected_abs": 0.05561278015375137, + "epoch": 0.060389352403655144, + "grad_norm": 0.8671875, + "learning_rate": 7.226939454625452e-08, + "logits/chosen": -2.314230442047119, + "logits/rejected": -2.4117889404296875, + "logps/chosen": -0.3054582476615906, + "logps/rejected": -0.3027587831020355, + "loss": 0.05369839817285538, + "loss/core": 0.050449974834918976, + "loss/preference_sft": 0.3041912019252777, + "loss/reference_anchor": 0.0623929388821125, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.245711088180542, + "rewards/margins": 0.6905202865600586, + "rewards/rejected": 0.5551908612251282, + "ronpo/logit": 0.06905203312635422, + "ronpo/residual": 0.062486421316862106, + "ronpo/residual_abs": 0.13185548782348633, + "ronpo/target": 0.00656561553478241, + "ronpo/target_abs": 0.08973196148872375, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.21235451102256775, + "drift/rejected_abs": 0.11808476597070694, + "epoch": 0.06197854588796186, + "grad_norm": 6.40625, + "learning_rate": 7.199045234496058e-08, + "logits/chosen": -2.054386615753174, + "logits/rejected": -2.0310075283050537, + "logps/chosen": -0.2783626914024353, + "logps/rejected": -0.27244699001312256, + "loss": 0.05899541452527046, + "loss/core": 0.05395280569791794, + "loss/preference_sft": 0.27698707580566406, + "loss/reference_anchor": 0.11637584865093231, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1214401721954346, + "rewards/margins": 0.9426848292350769, + "rewards/rejected": 1.1787551641464233, + "ronpo/logit": 0.09426848590373993, + "ronpo/residual": 0.09450878202915192, + "ronpo/residual_abs": 0.1405004858970642, + "ronpo/target": -0.00024029240012168884, + "ronpo/target_abs": 0.07883597910404205, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.2827044129371643, + "drift/rejected_abs": 0.08958513289690018, + "epoch": 0.06356773937226858, + "grad_norm": 8.875, + "learning_rate": 7.169853969509582e-08, + "logits/chosen": -2.036879777908325, + "logits/rejected": -2.1572928428649902, + "logps/chosen": -0.28990262746810913, + "logps/rejected": -0.2788967192173004, + "loss": 0.4873984456062317, + "loss/core": 0.4752472937107086, + "loss/preference_sft": 0.28781306743621826, + "loss/reference_anchor": 0.3183947503566742, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8262548446655273, + "rewards/margins": 1.9317960739135742, + "rewards/rejected": 0.8944588899612427, + "ronpo/logit": 0.19317962229251862, + "ronpo/residual": 0.19387605786323547, + "ronpo/residual_abs": 0.2424454241991043, + "ronpo/target": -0.000696455710567534, + "ronpo/target_abs": 0.0863131508231163, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.2943757474422455, + "drift/rejected_abs": 0.10825987160205841, + "epoch": 0.06515693285657528, + "grad_norm": 101.5, + "learning_rate": 7.139376637306952e-08, + "logits/chosen": -2.000290632247925, + "logits/rejected": -2.0813472270965576, + "logps/chosen": -0.2815471589565277, + "logps/rejected": -0.2869817614555359, + "loss": 0.7546241879463196, + "loss/core": 0.7346261739730835, + "loss/preference_sft": 0.287017822265625, + "loss/reference_anchor": 0.5426746606826782, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9414560794830322, + "rewards/margins": 1.859922170639038, + "rewards/rejected": 1.0815337896347046, + "ronpo/logit": 0.18599221110343933, + "ronpo/residual": 0.21781902015209198, + "ronpo/residual_abs": 0.2372894585132599, + "ronpo/target": -0.031826816499233246, + "ronpo/target_abs": 0.06753235310316086, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.21875524520874023, + "drift/rejected_abs": 0.058099307119846344, + "epoch": 0.066746126340882, + "grad_norm": 3.40625, + "learning_rate": 7.107624699166376e-08, + "logits/chosen": -2.2508599758148193, + "logits/rejected": -2.1692118644714355, + "logps/chosen": -0.3082744777202606, + "logps/rejected": -0.31492576003074646, + "loss": 1.0409514904022217, + "loss/core": 1.020789384841919, + "loss/preference_sft": 0.3110334277153015, + "loss/reference_anchor": 0.5449555516242981, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.1875526905059814, + "rewards/margins": 1.6086879968643188, + "rewards/rejected": 0.5788646340370178, + "ronpo/logit": 0.16086876392364502, + "ronpo/residual": 0.16442081332206726, + "ronpo/residual_abs": 0.21720726788043976, + "ronpo/target": -0.003552014008164406, + "ronpo/target_abs": 0.07966691255569458, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.32707658410072327, + "drift/rejected_abs": 0.164951890707016, + "epoch": 0.06833531982518871, + "grad_norm": 1.0234375, + "learning_rate": 7.074610095693229e-08, + "logits/chosen": -2.1667943000793457, + "logits/rejected": -2.154695510864258, + "logps/chosen": -0.26537930965423584, + "logps/rejected": -0.2718598246574402, + "loss": 0.29404348134994507, + "loss/core": 0.27320724725723267, + "loss/preference_sft": 0.2671200931072235, + "loss/reference_anchor": 0.5686091780662537, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.270766019821167, + "rewards/margins": 1.6218563318252563, + "rewards/rejected": 1.648909330368042, + "ronpo/logit": 0.1621856391429901, + "ronpo/residual": 0.16278550028800964, + "ronpo/residual_abs": 0.21330192685127258, + "ronpo/target": -0.0005998473498038948, + "ronpo/target_abs": 0.07392500340938568, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.11985316127538681, + "drift/rejected_abs": 0.08097579330205917, + "epoch": 0.06992451330949544, + "grad_norm": 0.482421875, + "learning_rate": 7.040345242329681e-08, + "logits/chosen": -2.214625120162964, + "logits/rejected": -2.082509756088257, + "logps/chosen": -0.30458956956863403, + "logps/rejected": -0.2999945878982544, + "loss": 0.06141216307878494, + "loss/core": 0.05872996523976326, + "loss/preference_sft": 0.302223265171051, + "loss/reference_anchor": 0.046411849558353424, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.1941266059875488, + "rewards/margins": 0.3879542350769043, + "rewards/rejected": 0.806172251701355, + "ronpo/logit": 0.03879542648792267, + "ronpo/residual": 0.04558854550123215, + "ronpo/residual_abs": 0.13173437118530273, + "ronpo/target": -0.006793119013309479, + "ronpo/target_abs": 0.07642895728349686, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.23159165680408478, + "drift/rejected_abs": 0.10313665866851807, + "epoch": 0.07151370679380215, + "grad_norm": 4.9375, + "learning_rate": 7.004843024685753e-08, + "logits/chosen": -2.178675651550293, + "logits/rejected": -2.195631504058838, + "logps/chosen": -0.2871968150138855, + "logps/rejected": -0.2853408753871918, + "loss": 0.20537880063056946, + "loss/core": 0.19829311966896057, + "loss/preference_sft": 0.28042349219322205, + "loss/reference_anchor": 0.1744067519903183, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.3159165382385254, + "rewards/margins": 1.2847979068756104, + "rewards/rejected": 1.031118631362915, + "ronpo/logit": 0.12847980856895447, + "ronpo/residual": 0.11979977041482925, + "ronpo/residual_abs": 0.16873392462730408, + "ronpo/target": 0.008680026978254318, + "ronpo/target_abs": 0.06639182567596436, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.26562970876693726, + "drift/rejected_abs": 0.0964142382144928, + "epoch": 0.07310290027810885, + "grad_norm": 2.578125, + "learning_rate": 6.968116793693563e-08, + "logits/chosen": -2.240947723388672, + "logits/rejected": -2.258277654647827, + "logps/chosen": -0.2617165446281433, + "logps/rejected": -0.26082921028137207, + "loss": 0.40380316972732544, + "loss/core": 0.3928808569908142, + "loss/preference_sft": 0.2601296007633209, + "loss/reference_anchor": 0.2860526442527771, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.656296730041504, + "rewards/margins": 1.6946455240249634, + "rewards/rejected": 0.9616514444351196, + "ronpo/logit": 0.16946455836296082, + "ronpo/residual": 0.16977901756763458, + "ronpo/residual_abs": 0.21348360180854797, + "ronpo/target": -0.0003144521324429661, + "ronpo/target_abs": 0.07685980945825577, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.46096473932266235, + "drift/rejected_abs": 0.2360442876815796, + "epoch": 0.07469209376241558, + "grad_norm": 63.25, + "learning_rate": 6.930180360586597e-08, + "logits/chosen": -2.1169967651367188, + "logits/rejected": -2.118549108505249, + "logps/chosen": -0.28103750944137573, + "logps/rejected": -0.28609365224838257, + "loss": 1.180558443069458, + "loss/core": 1.1377079486846924, + "loss/preference_sft": 0.28971725702285767, + "loss/reference_anchor": 1.1953245401382446, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.6081223487854, + "rewards/margins": 2.326866626739502, + "rewards/rejected": 2.2812554836273193, + "ronpo/logit": 0.23268663883209229, + "ronpo/residual": 0.2368365228176117, + "ronpo/residual_abs": 0.38683050870895386, + "ronpo/target": -0.0041498770006000996, + "ronpo/target_abs": 0.06436969339847565, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.29032057523727417, + "drift/rejected_abs": 0.16610702872276306, + "epoch": 0.07628128724672228, + "grad_norm": 169.0, + "learning_rate": 6.89104799170586e-08, + "logits/chosen": -2.1400163173675537, + "logits/rejected": -2.216646671295166, + "logps/chosen": -0.2673208713531494, + "logps/rejected": -0.2859800159931183, + "loss": 0.206389382481575, + "loss/core": 0.18720188736915588, + "loss/preference_sft": 0.2810494303703308, + "loss/reference_anchor": 0.520110011100769, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9032058715820312, + "rewards/margins": 1.2437211275100708, + "rewards/rejected": 1.6594845056533813, + "ronpo/logit": 0.12437212467193604, + "ronpo/residual": 0.10429469496011734, + "ronpo/residual_abs": 0.1717267632484436, + "ronpo/target": 0.020077422261238098, + "ronpo/target_abs": 0.07924573123455048, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.18758592009544373, + "drift/rejected_abs": 0.0732673853635788, + "epoch": 0.07787048073102901, + "grad_norm": 2.25, + "learning_rate": 6.85073440313489e-08, + "logits/chosen": -2.178632974624634, + "logits/rejected": -2.1970858573913574, + "logps/chosen": -0.29963111877441406, + "logps/rejected": -0.3051955997943878, + "loss": 0.2845226526260376, + "loss/core": 0.277618944644928, + "loss/preference_sft": 0.3048124611377716, + "loss/reference_anchor": 0.1667684018611908, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8743228912353516, + "rewards/margins": 1.169947624206543, + "rewards/rejected": 0.7043755054473877, + "ronpo/logit": 0.11699476093053818, + "ronpo/residual": 0.12534533441066742, + "ronpo/residual_abs": 0.17828436195850372, + "ronpo/target": -0.008350581862032413, + "ronpo/target_abs": 0.08381186425685883, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.36205315589904785, + "drift/rejected_abs": 0.0857582613825798, + "epoch": 0.07945967421533572, + "grad_norm": 7.6875, + "learning_rate": 6.809254755165645e-08, + "logits/chosen": -2.110821485519409, + "logits/rejected": -2.0873301029205322, + "logps/chosen": -0.30966055393218994, + "logps/rejected": -0.3097647726535797, + "loss": 1.2090619802474976, + "loss/core": 1.1840286254882812, + "loss/preference_sft": 0.3095891773700714, + "loss/reference_anchor": 0.6842803955078125, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.619492769241333, + "rewards/margins": 2.763693332672119, + "rewards/rejected": 0.8557994961738586, + "ronpo/logit": 0.2763693332672119, + "ronpo/residual": 0.2691985070705414, + "ronpo/residual_abs": 0.32348698377609253, + "ronpo/target": 0.007170822471380234, + "ronpo/target_abs": 0.07173763960599899, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.44590863585472107, + "drift/rejected_abs": 0.1995648592710495, + "epoch": 0.08104886769964244, + "grad_norm": 6.875, + "learning_rate": 6.766624646597332e-08, + "logits/chosen": -2.1040661334991455, + "logits/rejected": -2.106905937194824, + "logps/chosen": -0.26315388083457947, + "logps/rejected": -0.2701566815376282, + "loss": 0.7402136325836182, + "loss/core": 0.7059299349784851, + "loss/preference_sft": 0.27266380190849304, + "loss/reference_anchor": 0.9522676467895508, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.455777168273926, + "rewards/margins": 2.532688617706299, + "rewards/rejected": 1.9230890274047852, + "ronpo/logit": 0.25326889753341675, + "ronpo/residual": 0.2603941559791565, + "ronpo/residual_abs": 0.30658119916915894, + "ronpo/target": -0.007125276140868664, + "ronpo/target_abs": 0.08376964181661606, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.33060067892074585, + "drift/rejected_abs": 0.0853637307882309, + "epoch": 0.08263806118394915, + "grad_norm": 880.0, + "learning_rate": 6.722860108870342e-08, + "logits/chosen": -2.1947333812713623, + "logits/rejected": -2.1691174507141113, + "logps/chosen": -0.280915766954422, + "logps/rejected": -0.28027015924453735, + "loss": 0.8541151881217957, + "loss/core": 0.8351130485534668, + "loss/preference_sft": 0.2944938540458679, + "loss/reference_anchor": 0.5134744644165039, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.304910182952881, + "rewards/margins": 2.455112934112549, + "rewards/rejected": 0.849797248840332, + "ronpo/logit": 0.24551133811473846, + "ronpo/residual": 0.2461731731891632, + "ronpo/residual_abs": 0.2894217371940613, + "ronpo/target": -0.000661846250295639, + "ronpo/target_abs": 0.07702169567346573, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.18190790712833405, + "drift/rejected_abs": 0.08150045573711395, + "epoch": 0.08422725466825585, + "grad_norm": 16.0, + "learning_rate": 6.677977600037472e-08, + "logits/chosen": -2.1558258533477783, + "logits/rejected": -2.1550559997558594, + "logps/chosen": -0.2850190997123718, + "logps/rejected": -0.29064029455184937, + "loss": 0.0755200982093811, + "loss/core": 0.07134844362735748, + "loss/preference_sft": 0.28953614830970764, + "loss/reference_anchor": 0.0902366042137146, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8185592889785767, + "rewards/margins": 1.0096482038497925, + "rewards/rejected": 0.8089107275009155, + "ronpo/logit": 0.10096482932567596, + "ronpo/residual": 0.08930448442697525, + "ronpo/residual_abs": 0.14850127696990967, + "ronpo/target": 0.011660362593829632, + "ronpo/target_abs": 0.07749392092227936, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.29220855236053467, + "drift/rejected_abs": 0.16033557057380676, + "epoch": 0.08581644815256258, + "grad_norm": 2.328125, + "learning_rate": 6.631993998574737e-08, + "logits/chosen": -2.2243332862854004, + "logits/rejected": -2.183389663696289, + "logps/chosen": -0.2799707055091858, + "logps/rejected": -0.28995639085769653, + "loss": 0.3449793756008148, + "loss/core": 0.3193924129009247, + "loss/preference_sft": 0.28027012944221497, + "loss/reference_anchor": 0.7030280828475952, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.920807361602783, + "rewards/margins": 1.320322036743164, + "rewards/rejected": 1.60048508644104, + "ronpo/logit": 0.13203221559524536, + "ronpo/residual": 0.11397619545459747, + "ronpo/residual_abs": 0.1874753087759018, + "ronpo/target": 0.01805601641535759, + "ronpo/target_abs": 0.089533731341362, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.1789226233959198, + "drift/rejected_abs": 0.10433357954025269, + "epoch": 0.08740564163686929, + "grad_norm": 7.40625, + "learning_rate": 6.584926597034054e-08, + "logits/chosen": -2.2748570442199707, + "logits/rejected": -2.3492331504821777, + "logps/chosen": -0.27904680371284485, + "logps/rejected": -0.2677331864833832, + "loss": 0.04550812020897865, + "loss/core": 0.04103340208530426, + "loss/preference_sft": 0.27703091502189636, + "loss/reference_anchor": 0.10014583170413971, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.7892261743545532, + "rewards/margins": 0.7504833936691284, + "rewards/rejected": 1.0387427806854248, + "ronpo/logit": 0.07504834234714508, + "ronpo/residual": 0.06096462160348892, + "ronpo/residual_abs": 0.12806007266044617, + "ronpo/target": 0.014083716087043285, + "ronpo/target_abs": 0.07628627121448517, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.1692146509885788, + "drift/rejected_abs": 0.11026661098003387, + "epoch": 0.08899483512117601, + "grad_norm": 50.0, + "learning_rate": 6.536793095540228e-08, + "logits/chosen": -2.18296480178833, + "logits/rejected": -2.1851444244384766, + "logps/chosen": -0.2847527265548706, + "logps/rejected": -0.29109668731689453, + "loss": 0.08967806398868561, + "loss/core": 0.08531448990106583, + "loss/preference_sft": 0.28928548097610474, + "loss/reference_anchor": 0.09574474394321442, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6921466588974, + "rewards/margins": 0.5903476476669312, + "rewards/rejected": 1.1017990112304688, + "ronpo/logit": 0.05903477221727371, + "ronpo/residual": 0.04740891978144646, + "ronpo/residual_abs": 0.1459716111421585, + "ronpo/target": 0.011625855229794979, + "ronpo/target_abs": 0.067486971616745, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.4365122318267822, + "drift/rejected_abs": 0.14117935299873352, + "epoch": 0.09058402860548272, + "grad_norm": 9.875, + "learning_rate": 6.487611595134655e-08, + "logits/chosen": -2.2247390747070312, + "logits/rejected": -2.196688175201416, + "logps/chosen": -0.28154322504997253, + "logps/rejected": -0.2904406487941742, + "loss": 1.4670677185058594, + "loss/core": 1.4230749607086182, + "loss/preference_sft": 0.2900314927101135, + "loss/reference_anchor": 1.2279331684112549, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.364717960357666, + "rewards/margins": 2.953922986984253, + "rewards/rejected": 1.4107954502105713, + "ronpo/logit": 0.29539233446121216, + "ronpo/residual": 0.3029133677482605, + "ronpo/residual_abs": 0.355401873588562, + "ronpo/target": -0.0075210509821772575, + "ronpo/target_abs": 0.07652939856052399, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.299263060092926, + "drift/rejected_abs": 0.13971708714962006, + "epoch": 0.09217322208978942, + "grad_norm": 84.0, + "learning_rate": 6.43740059096826e-08, + "logits/chosen": -2.013880968093872, + "logits/rejected": -2.087587594985962, + "logps/chosen": -0.2723047435283661, + "logps/rejected": -0.28252094984054565, + "loss": 0.28749948740005493, + "loss/core": 0.2771393358707428, + "loss/preference_sft": 0.27838370203971863, + "loss/reference_anchor": 0.2681671977043152, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.991696357727051, + "rewards/margins": 1.5951488018035889, + "rewards/rejected": 1.396547555923462, + "ronpo/logit": 0.1595148742198944, + "ronpo/residual": 0.15534573793411255, + "ronpo/residual_abs": 0.23441383242607117, + "ronpo/target": 0.004169141408056021, + "ronpo/target_abs": 0.08591829240322113, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.19509822130203247, + "drift/rejected_abs": 0.07152483612298965, + "epoch": 0.09376241557409615, + "grad_norm": 192.0, + "learning_rate": 6.386178965346232e-08, + "logits/chosen": -2.1909687519073486, + "logits/rejected": -2.1839189529418945, + "logps/chosen": -0.30111661553382874, + "logps/rejected": -0.3057703375816345, + "loss": 0.2420537918806076, + "loss/core": 0.23458893597126007, + "loss/preference_sft": 0.29602617025375366, + "loss/reference_anchor": 0.18367911875247955, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9500179290771484, + "rewards/margins": 1.2395662069320679, + "rewards/rejected": 0.7104517817497253, + "ronpo/logit": 0.12395663559436798, + "ronpo/residual": 0.12118228524923325, + "ronpo/residual_abs": 0.16567404568195343, + "ronpo/target": 0.0027743394020944834, + "ronpo/target_abs": 0.06994090974330902, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.1367836445569992, + "drift/rejected_abs": 0.08203138411045074, + "epoch": 0.09535160905840286, + "grad_norm": 5.65625, + "learning_rate": 6.333965980627154e-08, + "logits/chosen": -2.4043996334075928, + "logits/rejected": -2.4086849689483643, + "logps/chosen": -0.29549726843833923, + "logps/rejected": -0.2862420082092285, + "loss": 0.030557457357645035, + "loss/core": 0.027756040915846825, + "loss/preference_sft": 0.29091423749923706, + "loss/reference_anchor": 0.05094895511865616, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3668571710586548, + "rewards/margins": 0.5511847734451294, + "rewards/rejected": 0.8156723976135254, + "ronpo/logit": 0.05511848255991936, + "ronpo/residual": 0.05358175188302994, + "ronpo/residual_abs": 0.10816502571105957, + "ronpo/target": 0.001536733703687787, + "ronpo/target_abs": 0.07785020023584366, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.22175483405590057, + "drift/rejected_abs": 0.10230524837970734, + "epoch": 0.09694080254270958, + "grad_norm": 2.90625, + "learning_rate": 6.280781271979229e-08, + "logits/chosen": -2.0780434608459473, + "logits/rejected": -2.1060025691986084, + "logps/chosen": -0.30538249015808105, + "logps/rejected": -0.29317212104797363, + "loss": 0.18119879066944122, + "loss/core": 0.17126640677452087, + "loss/preference_sft": 0.3047506511211395, + "loss/reference_anchor": 0.25330713391304016, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2160160541534424, + "rewards/margins": 1.1955595016479492, + "rewards/rejected": 1.0204565525054932, + "ronpo/logit": 0.11955595016479492, + "ronpo/residual": 0.12216754257678986, + "ronpo/residual_abs": 0.17102210223674774, + "ronpo/target": -0.0026116042863577604, + "ronpo/target_abs": 0.07656402885913849, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.229892298579216, + "drift/rejected_abs": 0.15156972408294678, + "epoch": 0.09852999602701629, + "grad_norm": 226.0, + "learning_rate": 6.226644839996278e-08, + "logits/chosen": -2.0961735248565674, + "logits/rejected": -1.9890098571777344, + "logps/chosen": -0.30494624376296997, + "logps/rejected": -0.29889440536499023, + "loss": 0.08074851334095001, + "loss/core": 0.06823646277189255, + "loss/preference_sft": 0.3070228397846222, + "loss/reference_anchor": 0.32678505778312683, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2979280948638916, + "rewards/margins": 0.7833226323127747, + "rewards/rejected": 1.5146055221557617, + "ronpo/logit": 0.07833226025104523, + "ronpo/residual": 0.0831359401345253, + "ronpo/residual_abs": 0.14295317232608795, + "ronpo/target": -0.0048036882653832436, + "ronpo/target_abs": 0.0696224644780159, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.21884290874004364, + "drift/rejected_abs": 0.1715756207704544, + "epoch": 0.10011918951132301, + "grad_norm": 1.96875, + "learning_rate": 6.171577043176358e-08, + "logits/chosen": -2.206495761871338, + "logits/rejected": -2.2399115562438965, + "logps/chosen": -0.2883833348751068, + "logps/rejected": -0.2834603190422058, + "loss": 0.11746679246425629, + "loss/core": 0.08842774480581284, + "loss/preference_sft": 0.2884564995765686, + "loss/reference_anchor": 0.8008415102958679, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1873815059661865, + "rewards/margins": 0.4734341502189636, + "rewards/rejected": 1.7139472961425781, + "ronpo/logit": 0.047343410551548004, + "ronpo/residual": 0.03924637287855148, + "ronpo/residual_abs": 0.13970692455768585, + "ronpo/target": 0.008097042329609394, + "ronpo/target_abs": 0.07850007712841034, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.30623868107795715, + "drift/rejected_abs": 0.15175890922546387, + "epoch": 0.10170838299562972, + "grad_norm": 5.15625, + "learning_rate": 6.115598590265742e-08, + "logits/chosen": -2.048631191253662, + "logits/rejected": -2.0478978157043457, + "logps/chosen": -0.2675008773803711, + "logps/rejected": -0.26748839020729065, + "loss": 0.41966015100479126, + "loss/core": 0.3942663371562958, + "loss/preference_sft": 0.26846352219581604, + "loss/reference_anchor": 0.6986923217773438, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.06076717376709, + "rewards/margins": 1.5459158420562744, + "rewards/rejected": 1.5148515701293945, + "ronpo/logit": 0.15459157526493073, + "ronpo/residual": 0.15290841460227966, + "ronpo/residual_abs": 0.209668830037117, + "ronpo/target": 0.0016831799875944853, + "ronpo/target_abs": 0.0813584178686142, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.2687004804611206, + "drift/rejected_abs": 0.06457342952489853, + "epoch": 0.10329757647993643, + "grad_norm": 22.5, + "learning_rate": 6.058730532471218e-08, + "logits/chosen": -2.212750196456909, + "logits/rejected": -2.2377688884735107, + "logps/chosen": -0.3067273497581482, + "logps/rejected": -0.2894396185874939, + "loss": 1.0871208906173706, + "loss/core": 1.0651031732559204, + "loss/preference_sft": 0.3019077479839325, + "loss/reference_anchor": 0.5988877415657043, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.686971426010132, + "rewards/margins": 2.045879602432251, + "rewards/rejected": 0.6410917043685913, + "ronpo/logit": 0.20458796620368958, + "ronpo/residual": 0.2081504613161087, + "ronpo/residual_abs": 0.24282391369342804, + "ronpo/target": -0.003562526311725378, + "ronpo/target_abs": 0.07360132783651352, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.25193846225738525, + "drift/rejected_abs": 0.1257767379283905, + "epoch": 0.10488676996424315, + "grad_norm": 15.5, + "learning_rate": 6.000994255543592e-08, + "logits/chosen": -2.3243191242218018, + "logits/rejected": -2.323913812637329, + "logps/chosen": -0.265578955411911, + "logps/rejected": -0.26127713918685913, + "loss": 0.5734900832176208, + "loss/core": 0.5604211091995239, + "loss/preference_sft": 0.27206915616989136, + "loss/reference_anchor": 0.34619513154029846, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5193843841552734, + "rewards/margins": 1.2678606510162354, + "rewards/rejected": 1.2515236139297485, + "ronpo/logit": 0.12678608298301697, + "ronpo/residual": 0.14241120219230652, + "ronpo/residual_abs": 0.2712676525115967, + "ronpo/target": -0.015625139698386192, + "ronpo/target_abs": 0.09194107353687286, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.4483170509338379, + "drift/rejected_abs": 0.2124291956424713, + "epoch": 0.10647596344854986, + "grad_norm": 13.0625, + "learning_rate": 5.942411471735374e-08, + "logits/chosen": -1.9917030334472656, + "logits/rejected": -1.9397776126861572, + "logps/chosen": -0.29203617572784424, + "logps/rejected": -0.2925509214401245, + "loss": 0.7571202516555786, + "loss/core": 0.7059234380722046, + "loss/preference_sft": 0.29065781831741333, + "loss/reference_anchor": 1.4337010383605957, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.481464385986328, + "rewards/margins": 2.3606128692626953, + "rewards/rejected": 2.120851516723633, + "ronpo/logit": 0.23606130480766296, + "ronpo/residual": 0.25377950072288513, + "ronpo/residual_abs": 0.3121553659439087, + "ronpo/target": -0.017718184739351273, + "ronpo/target_abs": 0.07344368100166321, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.45905232429504395, + "drift/rejected_abs": 0.16429868340492249, + "epoch": 0.10806515693285658, + "grad_norm": 82.0, + "learning_rate": 5.883004211635694e-08, + "logits/chosen": -1.8839290142059326, + "logits/rejected": -1.9421613216400146, + "logps/chosen": -0.28718405961990356, + "logps/rejected": -0.2739580273628235, + "loss": 0.9680835008621216, + "loss/core": 0.93462735414505, + "loss/preference_sft": 0.2868141829967499, + "loss/reference_anchor": 0.927208423614502, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.588636875152588, + "rewards/margins": 2.951829195022583, + "rewards/rejected": 1.6368080377578735, + "ronpo/logit": 0.29518288373947144, + "ronpo/residual": 0.2900192439556122, + "ronpo/residual_abs": 0.33183783292770386, + "ronpo/target": 0.005163692403584719, + "ronpo/target_abs": 0.06677814573049545, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.2457018792629242, + "drift/rejected_abs": 0.12155888229608536, + "epoch": 0.10965435041716329, + "grad_norm": 63.25, + "learning_rate": 5.8227948158854975e-08, + "logits/chosen": -2.178396463394165, + "logits/rejected": -2.251084566116333, + "logps/chosen": -0.27320748567581177, + "logps/rejected": -0.27947288751602173, + "loss": 0.1250918060541153, + "loss/core": 0.1128639206290245, + "loss/preference_sft": 0.28871798515319824, + "loss/reference_anchor": 0.3204962909221649, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4553937911987305, + "rewards/margins": 1.2922033071517944, + "rewards/rejected": 1.1631903648376465, + "ronpo/logit": 0.12922033667564392, + "ronpo/residual": 0.1374775469303131, + "ronpo/residual_abs": 0.1687041074037552, + "ronpo/target": -0.008257212117314339, + "ronpo/target_abs": 0.06108090281486511, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.41426244378089905, + "drift/rejected_abs": 0.1653062105178833, + "epoch": 0.11124354390147001, + "grad_norm": 1.578125, + "learning_rate": 5.761805926776142e-08, + "logits/chosen": -2.268873691558838, + "logits/rejected": -2.296691417694092, + "logps/chosen": -0.23340287804603577, + "logps/rejected": -0.24630527198314667, + "loss": 0.9605229496955872, + "loss/core": 0.9214657545089722, + "loss/preference_sft": 0.233280748128891, + "loss/reference_anchor": 1.0925902128219604, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.142511367797852, + "rewards/margins": 2.4894490242004395, + "rewards/rejected": 1.6530622243881226, + "ronpo/logit": 0.24894490838050842, + "ronpo/residual": 0.24280676245689392, + "ronpo/residual_abs": 0.2877592444419861, + "ronpo/target": 0.006138127762824297, + "ronpo/target_abs": 0.06590522080659866, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.28039026260375977, + "drift/rejected_abs": 0.10265352576971054, + "epoch": 0.11283273738577672, + "grad_norm": 18.25, + "learning_rate": 5.700060479734557e-08, + "logits/chosen": -2.1270534992218018, + "logits/rejected": -2.02483868598938, + "logps/chosen": -0.28407421708106995, + "logps/rejected": -0.27686986327171326, + "loss": 0.40096959471702576, + "loss/core": 0.3891247808933258, + "loss/preference_sft": 0.2740500569343567, + "loss/reference_anchor": 0.3110194802284241, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8039023876190186, + "rewards/margins": 1.780269980430603, + "rewards/rejected": 1.0236326456069946, + "ronpo/logit": 0.17802700400352478, + "ronpo/residual": 0.16591885685920715, + "ronpo/residual_abs": 0.22742481529712677, + "ronpo/target": 0.012108135037124157, + "ronpo/target_abs": 0.081517793238163, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.2177037000656128, + "drift/rejected_abs": 0.0887526124715805, + "epoch": 0.11442193087008343, + "grad_norm": 11.0, + "learning_rate": 5.6375816946981745e-08, + "logits/chosen": -2.2079660892486572, + "logits/rejected": -2.2729287147521973, + "logps/chosen": -0.26719698309898376, + "logps/rejected": -0.25824791193008423, + "loss": 0.08846821635961533, + "loss/core": 0.08424760401248932, + "loss/preference_sft": 0.26704221963882446, + "loss/reference_anchor": 0.09388500452041626, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1766204833984375, + "rewards/margins": 1.2892426252365112, + "rewards/rejected": 0.8873778581619263, + "ronpo/logit": 0.12892426550388336, + "ronpo/residual": 0.13182158768177032, + "ronpo/residual_abs": 0.17560000717639923, + "ronpo/target": -0.002897311467677355, + "ronpo/target_abs": 0.07039253413677216, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.2076272964477539, + "drift/rejected_abs": 0.13079994916915894, + "epoch": 0.11601112435439015, + "grad_norm": 2.203125, + "learning_rate": 5.574393067382849e-08, + "logits/chosen": -2.094231128692627, + "logits/rejected": -2.132718324661255, + "logps/chosen": -0.28609710931777954, + "logps/rejected": -0.3247900903224945, + "loss": 0.05453226715326309, + "loss/core": 0.040341801941394806, + "loss/preference_sft": 0.29053995013237, + "loss/reference_anchor": 0.37638789415359497, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0743491649627686, + "rewards/margins": 0.7902168035507202, + "rewards/rejected": 1.284132480621338, + "ronpo/logit": 0.07902167737483978, + "ronpo/residual": 0.0732402577996254, + "ronpo/residual_abs": 0.12813961505889893, + "ronpo/target": 0.005781428888440132, + "ronpo/target_abs": 0.08649028837680817, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.31887754797935486, + "drift/rejected_abs": 0.2154131680727005, + "epoch": 0.11760031783869686, + "grad_norm": 13.0625, + "learning_rate": 5.51051836044709e-08, + "logits/chosen": -2.1323890686035156, + "logits/rejected": -2.1665499210357666, + "logps/chosen": -0.2781253159046173, + "logps/rejected": -0.28154605627059937, + "loss": 0.14984655380249023, + "loss/core": 0.10850384086370468, + "loss/preference_sft": 0.28171804547309875, + "loss/reference_anchor": 1.1530487537384033, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1884636878967285, + "rewards/margins": 1.0351537466049194, + "rewards/rejected": 2.1533100605010986, + "ronpo/logit": 0.1035153716802597, + "ronpo/residual": 0.09588007628917694, + "ronpo/residual_abs": 0.16027481853961945, + "ronpo/target": 0.007635296322405338, + "ronpo/target_abs": 0.0714506059885025, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.21330508589744568, + "drift/rejected_abs": 0.09820286929607391, + "epoch": 0.11918951132300358, + "grad_norm": 19.25, + "learning_rate": 5.445981594555891e-08, + "logits/chosen": -2.2786364555358887, + "logits/rejected": -2.3025929927825928, + "logps/chosen": -0.29398036003112793, + "logps/rejected": -0.2887040376663208, + "loss": 0.14517758786678314, + "loss/core": 0.1390768587589264, + "loss/preference_sft": 0.2918010950088501, + "loss/reference_anchor": 0.14512670040130615, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.132155179977417, + "rewards/margins": 1.1507148742675781, + "rewards/rejected": 0.9814404249191284, + "ronpo/logit": 0.11507149785757065, + "ronpo/residual": 0.12740284204483032, + "ronpo/residual_abs": 0.17654451727867126, + "ronpo/target": -0.01233135536313057, + "ronpo/target_abs": 0.08002184331417084, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.2749318480491638, + "drift/rejected_abs": 0.10027341544628143, + "epoch": 0.12077870480731029, + "grad_norm": 0.65234375, + "learning_rate": 5.3808070393475435e-08, + "logits/chosen": -2.1572823524475098, + "logits/rejected": -2.1735129356384277, + "logps/chosen": -0.2892719805240631, + "logps/rejected": -0.28280091285705566, + "loss": 0.5006850361824036, + "loss/core": 0.4873785972595215, + "loss/preference_sft": 0.28779616951942444, + "loss/reference_anchor": 0.35140353441238403, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7480342388153076, + "rewards/margins": 1.75465989112854, + "rewards/rejected": 0.9933745265007019, + "ronpo/logit": 0.17546597123146057, + "ronpo/residual": 0.16056114435195923, + "ronpo/residual_abs": 0.2221776247024536, + "ronpo/target": 0.014904849231243134, + "ronpo/target_abs": 0.07894332706928253, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.2522042691707611, + "drift/rejected_abs": 0.1486947387456894, + "epoch": 0.12236789829161701, + "grad_norm": 62.75, + "learning_rate": 5.315019204306817e-08, + "logits/chosen": -2.2786965370178223, + "logits/rejected": -2.2488064765930176, + "logps/chosen": -0.27131175994873047, + "logps/rejected": -0.26389139890670776, + "loss": 0.17779920995235443, + "loss/core": 0.16905324161052704, + "loss/preference_sft": 0.26512569189071655, + "loss/reference_anchor": 0.22337183356285095, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.521192789077759, + "rewards/margins": 1.0361357927322388, + "rewards/rejected": 1.4850571155548096, + "ronpo/logit": 0.10361359268426895, + "ronpo/residual": 0.0915413573384285, + "ronpo/residual_abs": 0.15346641838550568, + "ronpo/target": 0.012072229757905006, + "ronpo/target_abs": 0.08567623794078827, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.2105921506881714, + "drift/rejected_abs": 0.12278739362955093, + "epoch": 0.12395709177592372, + "grad_norm": 3.0625, + "learning_rate": 5.248642829547946e-08, + "logits/chosen": -2.0767416954040527, + "logits/rejected": -2.117583751678467, + "logps/chosen": -0.3050832748413086, + "logps/rejected": -0.2946804165840149, + "loss": 0.08686129003763199, + "loss/core": 0.07733297348022461, + "loss/preference_sft": 0.2956315875053406, + "loss/reference_anchor": 0.2426745593547821, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1027276515960693, + "rewards/margins": 0.8792668581008911, + "rewards/rejected": 1.2234607934951782, + "ronpo/logit": 0.08792667090892792, + "ronpo/residual": 0.10677643120288849, + "ronpo/residual_abs": 0.141727477312088, + "ronpo/target": -0.018849752843379974, + "ronpo/target_abs": 0.07699684798717499, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.2143975794315338, + "drift/rejected_abs": 0.07005457580089569, + "epoch": 0.12554628526023043, + "grad_norm": 2.703125, + "learning_rate": 5.181702876510881e-08, + "logits/chosen": -2.1568331718444824, + "logits/rejected": -2.2212538719177246, + "logps/chosen": -0.3092118203639984, + "logps/rejected": -0.2957898676395416, + "loss": 0.3654123842716217, + "loss/core": 0.3564660847187042, + "loss/preference_sft": 0.30225998163223267, + "loss/reference_anchor": 0.2253817319869995, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.142920732498169, + "rewards/margins": 1.4426730871200562, + "rewards/rejected": 0.700247585773468, + "ronpo/logit": 0.14426730573177338, + "ronpo/residual": 0.14315283298492432, + "ronpo/residual_abs": 0.18677830696105957, + "ronpo/target": 0.0011144764721393585, + "ronpo/target_abs": 0.07045630365610123, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.19173476099967957, + "drift/rejected_abs": 0.08401096612215042, + "epoch": 0.12713547874453715, + "grad_norm": 9.1875, + "learning_rate": 5.114224518574318e-08, + "logits/chosen": -2.2294397354125977, + "logits/rejected": -2.318349599838257, + "logps/chosen": -0.25690674781799316, + "logps/rejected": -0.26110321283340454, + "loss": 0.07373060286045074, + "loss/core": 0.06859154254198074, + "loss/preference_sft": 0.2571546137332916, + "loss/reference_anchor": 0.12111492455005646, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.916455864906311, + "rewards/margins": 1.0823928117752075, + "rewards/rejected": 0.8340628743171692, + "ronpo/logit": 0.10823929309844971, + "ronpo/residual": 0.10606695711612701, + "ronpo/residual_abs": 0.15262378752231598, + "ronpo/target": 0.0021723308600485325, + "ronpo/target_abs": 0.07407379150390625, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.17076550424098969, + "drift/rejected_abs": 0.1465177685022354, + "epoch": 0.12872467222884387, + "grad_norm": 1.5859375, + "learning_rate": 5.046233131589012e-08, + "logits/chosen": -2.2917768955230713, + "logits/rejected": -2.300046443939209, + "logps/chosen": -0.29497990012168884, + "logps/rejected": -0.2882741093635559, + "loss": 0.28003552556037903, + "loss/core": 0.27095627784729004, + "loss/preference_sft": 0.28535428643226624, + "loss/reference_anchor": 0.2308715283870697, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7055221796035767, + "rewards/margins": 0.24134886264801025, + "rewards/rejected": 1.464173436164856, + "ronpo/logit": 0.024134885519742966, + "ronpo/residual": 0.025067275390028954, + "ronpo/residual_abs": 0.1801082193851471, + "ronpo/target": -0.0009323973208665848, + "ronpo/target_abs": 0.07486466318368912, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.18705379962921143, + "drift/rejected_abs": 0.05877493694424629, + "epoch": 0.13031386571315057, + "grad_norm": 0.94921875, + "learning_rate": 4.977754284334966e-08, + "logits/chosen": -2.2271358966827393, + "logits/rejected": -2.1403744220733643, + "logps/chosen": -0.28943097591400146, + "logps/rejected": -0.28723686933517456, + "loss": 0.34275034070014954, + "loss/core": 0.33544498682022095, + "loss/preference_sft": 0.2885206937789917, + "loss/reference_anchor": 0.1798722892999649, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8687069416046143, + "rewards/margins": 1.3043181896209717, + "rewards/rejected": 0.5643887519836426, + "ronpo/logit": 0.13043180108070374, + "ronpo/residual": 0.1327742636203766, + "ronpo/residual_abs": 0.1725582629442215, + "ronpo/target": -0.0023424329701811075, + "ronpo/target_abs": 0.07028114050626755, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.12771040201187134, + "drift/rejected_abs": 0.05937645584344864, + "epoch": 0.1319030591974573, + "grad_norm": 18.75, + "learning_rate": 4.9088137289060525e-08, + "logits/chosen": -2.2670741081237793, + "logits/rejected": -2.297637701034546, + "logps/chosen": -0.2853187322616577, + "logps/rejected": -0.27392154932022095, + "loss": 0.0417989157140255, + "loss/core": 0.03925047814846039, + "loss/preference_sft": 0.27748462557792664, + "loss/reference_anchor": 0.04506412148475647, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.277104139328003, + "rewards/margins": 0.6841433644294739, + "rewards/rejected": 0.5929605960845947, + "ronpo/logit": 0.06841433793306351, + "ronpo/residual": 0.06507981568574905, + "ronpo/residual_abs": 0.1218712329864502, + "ronpo/target": 0.003334520850330591, + "ronpo/target_abs": 0.07405976951122284, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.3893112242221832, + "drift/rejected_abs": 0.1252056360244751, + "epoch": 0.133492252681764, + "grad_norm": 1.9921875, + "learning_rate": 4.8394373910257054e-08, + "logits/chosen": -2.1123807430267334, + "logits/rejected": -2.231128692626953, + "logps/chosen": -0.2718810737133026, + "logps/rejected": -0.272949755191803, + "loss": 0.8291887044906616, + "loss/core": 0.8047215342521667, + "loss/preference_sft": 0.27599674463272095, + "loss/reference_anchor": 0.6714580059051514, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.8931126594543457, + "rewards/margins": 2.735173463821411, + "rewards/rejected": 1.1579387187957764, + "ronpo/logit": 0.27351734042167664, + "ronpo/residual": 0.26163390278816223, + "ronpo/residual_abs": 0.302009254693985, + "ronpo/target": 0.011883469298481941, + "ronpo/target_abs": 0.07862479239702225, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.42474645376205444, + "drift/rejected_abs": 0.13992993533611298, + "epoch": 0.1350814461660707, + "grad_norm": 304.0, + "learning_rate": 4.769651360297318e-08, + "logits/chosen": -2.1638529300689697, + "logits/rejected": -2.1579020023345947, + "logps/chosen": -0.29796314239501953, + "logps/rejected": -0.29541391134262085, + "loss": 1.2007313966751099, + "loss/core": 1.1638450622558594, + "loss/preference_sft": 0.29007643461227417, + "loss/reference_anchor": 1.0248842239379883, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.246431350708008, + "rewards/margins": 2.849513530731201, + "rewards/rejected": 1.3969179391860962, + "ronpo/logit": 0.284951388835907, + "ronpo/residual": 0.2653922438621521, + "ronpo/residual_abs": 0.3415285050868988, + "ronpo/target": 0.019559111446142197, + "ronpo/target_abs": 0.09040865302085876, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.18750756978988647, + "drift/rejected_abs": 0.09092601388692856, + "epoch": 0.13667063965037743, + "grad_norm": 11.375, + "learning_rate": 4.699481880393007e-08, + "logits/chosen": -2.430100679397583, + "logits/rejected": -2.416260242462158, + "logps/chosen": -0.27946940064430237, + "logps/rejected": -0.2753836512565613, + "loss": 0.09714354574680328, + "loss/core": 0.09125364571809769, + "loss/preference_sft": 0.273917555809021, + "loss/reference_anchor": 0.1408909112215042, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8739410638809204, + "rewards/margins": 0.9661051630973816, + "rewards/rejected": 0.9078359603881836, + "ronpo/logit": 0.09661051630973816, + "ronpo/residual": 0.090634785592556, + "ronpo/residual_abs": 0.14002564549446106, + "ronpo/target": 0.005975720938295126, + "ronpo/target_abs": 0.06694401055574417, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.1731262505054474, + "drift/rejected_abs": 0.0927744135260582, + "epoch": 0.13825983313468415, + "grad_norm": 11.875, + "learning_rate": 4.628955339184442e-08, + "logits/chosen": -2.182917356491089, + "logits/rejected": -2.1770148277282715, + "logps/chosen": -0.27976903319358826, + "logps/rejected": -0.281031996011734, + "loss": 0.04321626201272011, + "loss/core": 0.03948911279439926, + "loss/preference_sft": 0.27680492401123047, + "loss/reference_anchor": 0.07880944013595581, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7302322387695312, + "rewards/margins": 0.80370032787323, + "rewards/rejected": 0.9265319108963013, + "ronpo/logit": 0.08037003129720688, + "ronpo/residual": 0.06914183497428894, + "ronpo/residual_abs": 0.1280720978975296, + "ronpo/target": 0.011228211224079132, + "ronpo/target_abs": 0.06739668548107147, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.31899359822273254, + "drift/rejected_abs": 0.1331823766231537, + "epoch": 0.13984902661899087, + "grad_norm": 3.984375, + "learning_rate": 4.55809825881945e-08, + "logits/chosen": -2.2753958702087402, + "logits/rejected": -2.300555467605591, + "logps/chosen": -0.2945733666419983, + "logps/rejected": -0.29113486409187317, + "loss": 0.7341649532318115, + "loss/core": 0.7019937634468079, + "loss/preference_sft": 0.29861825704574585, + "loss/reference_anchor": 0.8893128633499146, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1885926723480225, + "rewards/margins": 1.86493718624115, + "rewards/rejected": 1.323655366897583, + "ronpo/logit": 0.18649372458457947, + "ronpo/residual": 0.19063273072242737, + "ronpo/residual_abs": 0.24889390170574188, + "ronpo/target": -0.004139023832976818, + "ronpo/target_abs": 0.08812376856803894, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.19011065363883972, + "drift/rejected_abs": 0.16861161589622498, + "epoch": 0.14143822010329757, + "grad_norm": 286.0, + "learning_rate": 4.486937285748113e-08, + "logits/chosen": -2.2313015460968018, + "logits/rejected": -2.1554415225982666, + "logps/chosen": -0.280648797750473, + "logps/rejected": -0.27201026678085327, + "loss": 0.22713658213615417, + "loss/core": 0.21747353672981262, + "loss/preference_sft": 0.2815045714378357, + "loss/reference_anchor": 0.24793705344200134, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9011064767837524, + "rewards/margins": 0.21516799926757812, + "rewards/rejected": 1.6859385967254639, + "ronpo/logit": 0.021516799926757812, + "ronpo/residual": 0.026197385042905807, + "ronpo/residual_abs": 0.18634097278118134, + "ronpo/target": -0.0046805874444544315, + "ronpo/target_abs": 0.0748194083571434, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.3207959234714508, + "drift/rejected_abs": 0.1021362692117691, + "epoch": 0.1430274135876043, + "grad_norm": 31.25, + "learning_rate": 4.415499180702132e-08, + "logits/chosen": -2.263270854949951, + "logits/rejected": -2.3271467685699463, + "logps/chosen": -0.2672632336616516, + "logps/rejected": -0.2835923731327057, + "loss": 0.7797986268997192, + "loss/core": 0.7587357759475708, + "loss/preference_sft": 0.2777632474899292, + "loss/reference_anchor": 0.5740165114402771, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2072951793670654, + "rewards/margins": 2.1891021728515625, + "rewards/rejected": 1.018193006515503, + "ronpo/logit": 0.21891024708747864, + "ronpo/residual": 0.228356271982193, + "ronpo/residual_abs": 0.27203264832496643, + "ronpo/target": -0.00944601558148861, + "ronpo/target_abs": 0.06918533146381378, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.2078581303358078, + "drift/rejected_abs": 0.09944683313369751, + "epoch": 0.144616607071911, + "grad_norm": 9.875, + "learning_rate": 4.3438108086312104e-08, + "logits/chosen": -2.315324306488037, + "logits/rejected": -2.3483715057373047, + "logps/chosen": -0.2843065857887268, + "logps/rejected": -0.2822841703891754, + "loss": 0.1076749935746193, + "loss/core": 0.10172615200281143, + "loss/preference_sft": 0.2944115698337555, + "loss/reference_anchor": 0.14052584767341614, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0785813331604004, + "rewards/margins": 1.0842633247375488, + "rewards/rejected": 0.994317889213562, + "ronpo/logit": 0.10842633247375488, + "ronpo/residual": 0.10938017070293427, + "ronpo/residual_abs": 0.16080966591835022, + "ronpo/target": -0.0009538428857922554, + "ronpo/target_abs": 0.07275767624378204, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.37791651487350464, + "drift/rejected_abs": 0.16409462690353394, + "epoch": 0.1462058005562177, + "grad_norm": 115.5, + "learning_rate": 4.271899128600245e-08, + "logits/chosen": -1.805584192276001, + "logits/rejected": -1.6658241748809814, + "logps/chosen": -0.2807144224643707, + "logps/rejected": -0.30006617307662964, + "loss": 0.5550448298454285, + "loss/core": 0.5333111882209778, + "loss/preference_sft": 0.29056912660598755, + "loss/reference_anchor": 0.5919061303138733, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7732338905334473, + "rewards/margins": 2.1390435695648193, + "rewards/rejected": 1.634190320968628, + "ronpo/logit": 0.21390438079833984, + "ronpo/residual": 0.2101598083972931, + "ronpo/residual_abs": 0.26126962900161743, + "ronpo/target": 0.0037445551715791225, + "ronpo/target_abs": 0.07518764585256577, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.3444373905658722, + "drift/rejected_abs": 0.23399516940116882, + "epoch": 0.14779499404052443, + "grad_norm": 318.0, + "learning_rate": 4.199791183651124e-08, + "logits/chosen": -2.0957283973693848, + "logits/rejected": -2.116079092025757, + "logps/chosen": -0.2696365416049957, + "logps/rejected": -0.2735532522201538, + "loss": 0.8012321591377258, + "loss/core": 0.7794309854507446, + "loss/preference_sft": 0.277563214302063, + "loss/reference_anchor": 0.5951352119445801, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4441075325012207, + "rewards/margins": 1.1061354875564575, + "rewards/rejected": 2.3379721641540527, + "ronpo/logit": 0.11061354726552963, + "ronpo/residual": 0.099038265645504, + "ronpo/residual_abs": 0.29626208543777466, + "ronpo/target": 0.011575284413993359, + "ronpo/target_abs": 0.06793732196092606, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.29984086751937866, + "drift/rejected_abs": 0.11699891090393066, + "epoch": 0.14938418752483115, + "grad_norm": 2.640625, + "learning_rate": 4.127514090632942e-08, + "logits/chosen": -2.075214147567749, + "logits/rejected": -2.0434682369232178, + "logps/chosen": -0.254089891910553, + "logps/rejected": -0.2596934139728546, + "loss": 0.7762528657913208, + "loss/core": 0.7569972276687622, + "loss/preference_sft": 0.2593012750148773, + "loss/reference_anchor": 0.5242301821708679, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.996891975402832, + "rewards/margins": 1.8279199600219727, + "rewards/rejected": 1.1689717769622803, + "ronpo/logit": 0.18279200792312622, + "ronpo/residual": 0.1851007044315338, + "ronpo/residual_abs": 0.25966882705688477, + "ronpo/target": -0.0023087128065526485, + "ronpo/target_abs": 0.07931704819202423, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.25055521726608276, + "drift/rejected_abs": 0.09638573229312897, + "epoch": 0.15097338100913787, + "grad_norm": 8.3125, + "learning_rate": 4.055095030004469e-08, + "logits/chosen": -2.195185661315918, + "logits/rejected": -2.1315507888793945, + "logps/chosen": -0.30518704652786255, + "logps/rejected": -0.29946842789649963, + "loss": 0.2174142301082611, + "loss/core": 0.21045875549316406, + "loss/preference_sft": 0.3060879111289978, + "loss/reference_anchor": 0.16811935603618622, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5054726600646973, + "rewards/margins": 1.542541265487671, + "rewards/rejected": 0.9629314541816711, + "ronpo/logit": 0.15425412356853485, + "ronpo/residual": 0.15975168347358704, + "ronpo/residual_abs": 0.21089980006217957, + "ronpo/target": -0.0054975529201328754, + "ronpo/target_abs": 0.07147940248250961, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.22197389602661133, + "drift/rejected_abs": 0.1093837171792984, + "epoch": 0.15256257449344457, + "grad_norm": 1.5390625, + "learning_rate": 3.982561235612682e-08, + "logits/chosen": -2.1202902793884277, + "logits/rejected": -2.142862319946289, + "logps/chosen": -0.2731131911277771, + "logps/rejected": -0.2682819962501526, + "loss": 0.08932836353778839, + "loss/core": 0.08391264826059341, + "loss/preference_sft": 0.2691730856895447, + "loss/reference_anchor": 0.12781760096549988, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2168147563934326, + "rewards/margins": 1.1324942111968994, + "rewards/rejected": 1.0843205451965332, + "ronpo/logit": 0.11324940621852875, + "ronpo/residual": 0.13424549996852875, + "ronpo/residual_abs": 0.1686258614063263, + "ronpo/target": -0.020996104925870895, + "ronpo/target_abs": 0.07291201502084732, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.29734545946121216, + "drift/rejected_abs": 0.1604335755109787, + "epoch": 0.1541517679777513, + "grad_norm": 6.125, + "learning_rate": 3.90993998445124e-08, + "logits/chosen": -1.9032106399536133, + "logits/rejected": -2.025702953338623, + "logps/chosen": -0.28302717208862305, + "logps/rejected": -0.28014808893203735, + "loss": 0.12766507267951965, + "loss/core": 0.1155250072479248, + "loss/preference_sft": 0.28657975792884827, + "loss/reference_anchor": 0.3182009756565094, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.972499370574951, + "rewards/margins": 1.370253086090088, + "rewards/rejected": 1.6022460460662842, + "ronpo/logit": 0.13702531158924103, + "ronpo/residual": 0.1244589239358902, + "ronpo/residual_abs": 0.1808883100748062, + "ronpo/target": 0.012566402554512024, + "ronpo/target_abs": 0.08073397725820541, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.36853092908859253, + "drift/rejected_abs": 0.15083953738212585, + "epoch": 0.15574096146205801, + "grad_norm": 320.0, + "learning_rate": 3.837258586402711e-08, + "logits/chosen": -2.1028637886047363, + "logits/rejected": -2.0833709239959717, + "logps/chosen": -0.2817224860191345, + "logps/rejected": -0.2832174301147461, + "loss": 0.6819409132003784, + "loss/core": 0.6572893857955933, + "loss/preference_sft": 0.2822495102882385, + "loss/reference_anchor": 0.6761033535003662, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6840882301330566, + "rewards/margins": 2.1773343086242676, + "rewards/rejected": 1.50675368309021, + "ronpo/logit": 0.2177334576845169, + "ronpo/residual": 0.2175024449825287, + "ronpo/residual_abs": 0.26726433634757996, + "ronpo/target": 0.00023099928512237966, + "ronpo/target_abs": 0.07836781442165375, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.22081363201141357, + "drift/rejected_abs": 0.18465587496757507, + "epoch": 0.1573301549463647, + "grad_norm": 378.0, + "learning_rate": 3.7645443739684514e-08, + "logits/chosen": -2.126253843307495, + "logits/rejected": -2.0972957611083984, + "logps/chosen": -0.2862113416194916, + "logps/rejected": -0.2739894986152649, + "loss": 1.195561170578003, + "loss/core": 1.1711348295211792, + "loss/preference_sft": 0.2842695713043213, + "loss/reference_anchor": 0.6694715619087219, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2055563926696777, + "rewards/margins": 0.3634433150291443, + "rewards/rejected": 1.8421132564544678, + "ronpo/logit": 0.03634430468082428, + "ronpo/residual": 0.03679324686527252, + "ronpo/residual_abs": 0.3129720687866211, + "ronpo/target": -0.00044894107850268483, + "ronpo/target_abs": 0.0857948586344719, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.2433653324842453, + "drift/rejected_abs": 0.14672759175300598, + "epoch": 0.15891934843067143, + "grad_norm": 3.578125, + "learning_rate": 3.6918246919899624e-08, + "logits/chosen": -2.231562376022339, + "logits/rejected": -2.089289903640747, + "logps/chosen": -0.2816031575202942, + "logps/rejected": -0.2773313522338867, + "loss": 0.23030047118663788, + "loss/core": 0.21950602531433105, + "loss/preference_sft": 0.27701348066329956, + "loss/reference_anchor": 0.28071126341819763, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4336533546447754, + "rewards/margins": 0.9874534606933594, + "rewards/rejected": 1.446199893951416, + "ronpo/logit": 0.09874534606933594, + "ronpo/residual": 0.10226426273584366, + "ronpo/residual_abs": 0.20653577148914337, + "ronpo/target": -0.0035189155023545027, + "ronpo/target_abs": 0.07701299339532852, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.16075539588928223, + "drift/rejected_abs": 0.1068691611289978, + "epoch": 0.16050854191497815, + "grad_norm": 4.40625, + "learning_rate": 3.619126887365622e-08, + "logits/chosen": -2.2844841480255127, + "logits/rejected": -2.2047648429870605, + "logps/chosen": -0.2954888343811035, + "logps/rejected": -0.280442476272583, + "loss": 0.028839338570833206, + "loss/core": 0.025353241711854935, + "loss/preference_sft": 0.29189762473106384, + "loss/reference_anchor": 0.0704130083322525, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6047475337982178, + "rewards/margins": 0.536501944065094, + "rewards/rejected": 1.0682456493377686, + "ronpo/logit": 0.05365020036697388, + "ronpo/residual": 0.03312177211046219, + "ronpo/residual_abs": 0.1087198257446289, + "ronpo/target": 0.020528428256511688, + "ronpo/target_abs": 0.0795794278383255, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.3148565888404846, + "drift/rejected_abs": 0.16070185601711273, + "epoch": 0.16209773539928488, + "grad_norm": 2.609375, + "learning_rate": 3.5464782987666235e-08, + "logits/chosen": -1.940596342086792, + "logits/rejected": -1.9633086919784546, + "logps/chosen": -0.26210013031959534, + "logps/rejected": -0.26328200101852417, + "loss": 0.426157146692276, + "loss/core": 0.40358486771583557, + "loss/preference_sft": 0.263058602809906, + "loss/reference_anchor": 0.6186167597770691, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1481029987335205, + "rewards/margins": 1.5438060760498047, + "rewards/rejected": 1.6042969226837158, + "ronpo/logit": 0.15438061952590942, + "ronpo/residual": 0.15045881271362305, + "ronpo/residual_abs": 0.1978466957807541, + "ronpo/target": 0.00392179936170578, + "ronpo/target_abs": 0.06807820498943329, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.2575669288635254, + "drift/rejected_abs": 0.15305769443511963, + "epoch": 0.16368692888359157, + "grad_norm": 5.75, + "learning_rate": 3.473906246356032e-08, + "logits/chosen": -2.141854763031006, + "logits/rejected": -2.2106332778930664, + "logps/chosen": -0.2897399663925171, + "logps/rejected": -0.27617400884628296, + "loss": 0.3367363512516022, + "loss/core": 0.32625168561935425, + "loss/preference_sft": 0.28051096200942993, + "loss/reference_anchor": 0.2715080976486206, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.574777126312256, + "rewards/margins": 1.0462206602096558, + "rewards/rejected": 1.528556227684021, + "ronpo/logit": 0.10462206602096558, + "ronpo/residual": 0.09638858586549759, + "ronpo/residual_abs": 0.214156836271286, + "ronpo/target": 0.008233482949435711, + "ronpo/target_abs": 0.07465226948261261, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.21658673882484436, + "drift/rejected_abs": 0.10446375608444214, + "epoch": 0.1652761223678983, + "grad_norm": 2.0625, + "learning_rate": 3.401438021514779e-08, + "logits/chosen": -2.1490402221679688, + "logits/rejected": -2.1608524322509766, + "logps/chosen": -0.29304903745651245, + "logps/rejected": -0.3020794689655304, + "loss": 0.2509724795818329, + "loss/core": 0.23218867182731628, + "loss/preference_sft": 0.2949676513671875, + "loss/reference_anchor": 0.5071809887886047, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.165867328643799, + "rewards/margins": 1.1303069591522217, + "rewards/rejected": 1.0355603694915771, + "ronpo/logit": 0.11303069442510605, + "ronpo/residual": 0.10117007791996002, + "ronpo/residual_abs": 0.1607782542705536, + "ronpo/target": 0.011860623024404049, + "ronpo/target_abs": 0.08329831808805466, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.2207900583744049, + "drift/rejected_abs": 0.06476399302482605, + "epoch": 0.16686531585220502, + "grad_norm": 7.625, + "learning_rate": 3.329100876578495e-08, + "logits/chosen": -1.9039376974105835, + "logits/rejected": -2.0480172634124756, + "logps/chosen": -0.3111041188240051, + "logps/rejected": -0.3204375207424164, + "loss": 0.27312102913856506, + "loss/core": 0.2654520869255066, + "loss/preference_sft": 0.32211512327194214, + "loss/reference_anchor": 0.18690235912799835, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2061574459075928, + "rewards/margins": 1.5601660013198853, + "rewards/rejected": 0.6459915637969971, + "ronpo/logit": 0.15601658821105957, + "ronpo/residual": 0.16684645414352417, + "ronpo/residual_abs": 0.20652273297309875, + "ronpo/target": -0.010829855687916279, + "ronpo/target_abs": 0.08058717846870422, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.1540854275226593, + "drift/rejected_abs": 0.06486465036869049, + "epoch": 0.1684545093365117, + "grad_norm": 1.3046875, + "learning_rate": 3.256922014589015e-08, + "logits/chosen": -2.324816942214966, + "logits/rejected": -2.319702625274658, + "logps/chosen": -0.26595860719680786, + "logps/rejected": -0.26108628511428833, + "loss": 0.06081946939229965, + "loss/core": 0.05783188343048096, + "loss/preference_sft": 0.26668626070022583, + "loss/reference_anchor": 0.05869119241833687, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.5408542156219482, + "rewards/margins": 0.8943354487419128, + "rewards/rejected": 0.6465188264846802, + "ronpo/logit": 0.08943354338407516, + "ronpo/residual": 0.09258811175823212, + "ronpo/residual_abs": 0.13264910876750946, + "ronpo/target": -0.003154561622068286, + "ronpo/target_abs": 0.0733359232544899, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.3481886386871338, + "drift/rejected_abs": 0.14496779441833496, + "epoch": 0.17004370282081843, + "grad_norm": 2.40625, + "learning_rate": 3.184928579064438e-08, + "logits/chosen": -2.240485429763794, + "logits/rejected": -2.2552919387817383, + "logps/chosen": -0.3145575523376465, + "logps/rejected": -0.3206997513771057, + "loss": 1.516591191291809, + "loss/core": 1.477942705154419, + "loss/preference_sft": 0.31602850556373596, + "loss/reference_anchor": 1.0726332664489746, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4811675548553467, + "rewards/margins": 2.195894241333008, + "rewards/rejected": 1.2852733135223389, + "ronpo/logit": 0.21958939731121063, + "ronpo/residual": 0.23421013355255127, + "ronpo/residual_abs": 0.340684711933136, + "ronpo/target": -0.014620688743889332, + "ronpo/target_abs": 0.08300719410181046, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.2462986409664154, + "drift/rejected_abs": 0.10847475379705429, + "epoch": 0.17163289630512515, + "grad_norm": 3.234375, + "learning_rate": 3.1131476437915473e-08, + "logits/chosen": -2.202176570892334, + "logits/rejected": -2.2374606132507324, + "logps/chosen": -0.27796033024787903, + "logps/rejected": -0.2893964350223541, + "loss": 0.22307059168815613, + "loss/core": 0.20895227789878845, + "loss/preference_sft": 0.28652340173721313, + "loss/reference_anchor": 0.3747285306453705, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.462252140045166, + "rewards/margins": 1.377599835395813, + "rewards/rejected": 1.084652304649353, + "ronpo/logit": 0.1377600133419037, + "ronpo/residual": 0.13737379014492035, + "ronpo/residual_abs": 0.19108590483665466, + "ronpo/target": 0.0003861982258968055, + "ronpo/target_abs": 0.06714820116758347, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.39844200015068054, + "drift/rejected_abs": 0.2459792196750641, + "epoch": 0.17322208978943188, + "grad_norm": 5.1875, + "learning_rate": 3.0416062026444745e-08, + "logits/chosen": -1.9908326864242554, + "logits/rejected": -2.015047550201416, + "logps/chosen": -0.26075753569602966, + "logps/rejected": -0.27910637855529785, + "loss": 0.45975977182388306, + "loss/core": 0.42998823523521423, + "loss/preference_sft": 0.2777344584465027, + "loss/reference_anchor": 0.8228422999382019, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.9832959175109863, + "rewards/margins": 1.527392029762268, + "rewards/rejected": 2.455904006958008, + "ronpo/logit": 0.15273919701576233, + "ronpo/residual": 0.1590009331703186, + "ronpo/residual_abs": 0.25908511877059937, + "ronpo/target": -0.006261733826249838, + "ronpo/target_abs": 0.08037153631448746, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.2750501334667206, + "drift/rejected_abs": 0.13786940276622772, + "epoch": 0.17481128327373857, + "grad_norm": 1.1015625, + "learning_rate": 2.970331159433403e-08, + "logits/chosen": -2.1441848278045654, + "logits/rejected": -2.100531578063965, + "logps/chosen": -0.25560611486434937, + "logps/rejected": -0.26517319679260254, + "loss": 0.3065684139728546, + "loss/core": 0.2939642667770386, + "loss/preference_sft": 0.2637214660644531, + "loss/reference_anchor": 0.3337470293045044, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.750082492828369, + "rewards/margins": 1.3717514276504517, + "rewards/rejected": 1.378330945968628, + "ronpo/logit": 0.13717512786388397, + "ronpo/residual": 0.16072766482830048, + "ronpo/residual_abs": 0.20835892856121063, + "ronpo/target": -0.023552536964416504, + "ronpo/target_abs": 0.08319397270679474, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.12603361904621124, + "drift/rejected_abs": 0.05538574978709221, + "epoch": 0.1764004767580453, + "grad_norm": 0.87109375, + "learning_rate": 2.8993493177871495e-08, + "logits/chosen": -2.215672016143799, + "logits/rejected": -2.2594668865203857, + "logps/chosen": -0.31073468923568726, + "logps/rejected": -0.30866894125938416, + "loss": 0.06362307071685791, + "loss/core": 0.05991923063993454, + "loss/preference_sft": 0.3092317581176758, + "loss/reference_anchor": 0.07490097731351852, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2597297430038452, + "rewards/margins": 0.7066539525985718, + "rewards/rejected": 0.5530757308006287, + "ronpo/logit": 0.0706654042005539, + "ronpo/residual": 0.05782570317387581, + "ronpo/residual_abs": 0.1277032196521759, + "ronpo/target": 0.012839692644774914, + "ronpo/target_abs": 0.08341912925243378, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.26530271768569946, + "drift/rejected_abs": 0.12235991656780243, + "epoch": 0.17798967024235202, + "grad_norm": 17.25, + "learning_rate": 2.8286873710734132e-08, + "logits/chosen": -2.085667133331299, + "logits/rejected": -2.0912811756134033, + "logps/chosen": -0.27223190665245056, + "logps/rejected": -0.2789652347564697, + "loss": 0.25534918904304504, + "loss/core": 0.24563512206077576, + "loss/preference_sft": 0.2783053517341614, + "loss/reference_anchor": 0.24971409142017365, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.650958299636841, + "rewards/margins": 1.4280980825424194, + "rewards/rejected": 1.2228600978851318, + "ronpo/logit": 0.14280982315540314, + "ronpo/residual": 0.14116224646568298, + "ronpo/residual_abs": 0.1777869462966919, + "ronpo/target": 0.0016475621378049254, + "ronpo/target_abs": 0.06457264721393585, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.2612389326095581, + "drift/rejected_abs": 0.11688123643398285, + "epoch": 0.1795788637266587, + "grad_norm": 30.75, + "learning_rate": 2.7583718923604928e-08, + "logits/chosen": -2.042419910430908, + "logits/rejected": -2.0682761669158936, + "logps/chosen": -0.33960863947868347, + "logps/rejected": -0.3336796164512634, + "loss": 0.359559029340744, + "loss/core": 0.34349602460861206, + "loss/preference_sft": 0.34128159284591675, + "loss/reference_anchor": 0.4248151183128357, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6094517707824707, + "rewards/margins": 1.4438995122909546, + "rewards/rejected": 1.1655519008636475, + "ronpo/logit": 0.14438995718955994, + "ronpo/residual": 0.13987357914447784, + "ronpo/residual_abs": 0.19552505016326904, + "ronpo/target": 0.004516408313065767, + "ronpo/target_abs": 0.07334297150373459, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.4135201871395111, + "drift/rejected_abs": 0.09967617690563202, + "epoch": 0.18116805721096543, + "grad_norm": 7.78125, + "learning_rate": 2.688429324424236e-08, + "logits/chosen": -2.1476290225982666, + "logits/rejected": -2.1181139945983887, + "logps/chosen": -0.27395740151405334, + "logps/rejected": -0.2648341655731201, + "loss": 1.3773685693740845, + "loss/core": 1.3479124307632446, + "loss/preference_sft": 0.27103132009506226, + "loss/reference_anchor": 0.8145006895065308, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.134523868560791, + "rewards/margins": 3.1398844718933105, + "rewards/rejected": 0.99463951587677, + "ronpo/logit": 0.31398844718933105, + "ronpo/residual": 0.30398809909820557, + "ronpo/residual_abs": 0.3727145493030548, + "ronpo/target": 0.010000362992286682, + "ronpo/target_abs": 0.09326197952032089, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.19183699786663055, + "drift/rejected_abs": 0.09668353945016861, + "epoch": 0.18275725069527216, + "grad_norm": 3.484375, + "learning_rate": 2.6188859698040042e-08, + "logits/chosen": -2.0081419944763184, + "logits/rejected": -2.0951578617095947, + "logps/chosen": -0.2836478650569916, + "logps/rejected": -0.27917319536209106, + "loss": 0.0594443753361702, + "loss/core": 0.05546221882104874, + "loss/preference_sft": 0.2888682782649994, + "loss/reference_anchor": 0.08488909900188446, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9143584966659546, + "rewards/margins": 0.9508224725723267, + "rewards/rejected": 0.9635359048843384, + "ronpo/logit": 0.09508226811885834, + "ronpo/residual": 0.10423971712589264, + "ronpo/residual_abs": 0.1455569565296173, + "ronpo/target": -0.009157449007034302, + "ronpo/target_abs": 0.07435912638902664, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.3637532591819763, + "drift/rejected_abs": 0.16664664447307587, + "epoch": 0.18434644417957885, + "grad_norm": 1.4453125, + "learning_rate": 2.5497679809113563e-08, + "logits/chosen": -2.001953125, + "logits/rejected": -2.0324103832244873, + "logps/chosen": -0.2893816828727722, + "logps/rejected": -0.2681158483028412, + "loss": 0.548211395740509, + "loss/core": 0.5249797105789185, + "loss/preference_sft": 0.28467053174972534, + "loss/reference_anchor": 0.6352949738502502, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6368331909179688, + "rewards/margins": 1.9708592891693115, + "rewards/rejected": 1.6659743785858154, + "ronpo/logit": 0.1970859318971634, + "ronpo/residual": 0.1857091784477234, + "ronpo/residual_abs": 0.2529819905757904, + "ronpo/target": 0.011376758106052876, + "ronpo/target_abs": 0.06672044098377228, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.23147885501384735, + "drift/rejected_abs": 0.09522049129009247, + "epoch": 0.18593563766388557, + "grad_norm": 1.8203125, + "learning_rate": 2.481101350195196e-08, + "logits/chosen": -1.9583909511566162, + "logits/rejected": -1.982860803604126, + "logps/chosen": -0.28692710399627686, + "logps/rejected": -0.2864198684692383, + "loss": 0.15366187691688538, + "loss/core": 0.14675399661064148, + "loss/preference_sft": 0.2956783175468445, + "loss/reference_anchor": 0.16780035197734833, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.313133955001831, + "rewards/margins": 1.362217903137207, + "rewards/rejected": 0.9509159922599792, + "ronpo/logit": 0.13622179627418518, + "ronpo/residual": 0.13147194683551788, + "ronpo/residual_abs": 0.17332948744297028, + "ronpo/target": 0.004749844316393137, + "ronpo/target_abs": 0.06973059475421906, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.259533554315567, + "drift/rejected_abs": 0.12255913019180298, + "epoch": 0.1875248311481923, + "grad_norm": 1.3671875, + "learning_rate": 2.4129119003670752e-08, + "logits/chosen": -2.1177847385406494, + "logits/rejected": -2.0852158069610596, + "logps/chosen": -0.3071213364601135, + "logps/rejected": -0.31466570496559143, + "loss": 0.545588493347168, + "loss/core": 0.5284724831581116, + "loss/preference_sft": 0.315735787153244, + "loss/reference_anchor": 0.4574582576751709, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.592897891998291, + "rewards/margins": 1.3681689500808716, + "rewards/rejected": 1.2247288227081299, + "ronpo/logit": 0.13681688904762268, + "ronpo/residual": 0.14628082513809204, + "ronpo/residual_abs": 0.19750835001468658, + "ronpo/target": -0.009463920257985592, + "ronpo/target_abs": 0.07016940414905548, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.23209957778453827, + "drift/rejected_abs": 0.11176522821187973, + "epoch": 0.18911402463249902, + "grad_norm": 5.84375, + "learning_rate": 2.3452252746903295e-08, + "logits/chosen": -1.974440336227417, + "logits/rejected": -1.9321467876434326, + "logps/chosen": -0.28932034969329834, + "logps/rejected": -0.29376018047332764, + "loss": 0.1980138123035431, + "loss/core": 0.19030292332172394, + "loss/preference_sft": 0.29327264428138733, + "loss/reference_anchor": 0.19098426401615143, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3200583457946777, + "rewards/margins": 1.2056306600570679, + "rewards/rejected": 1.1144278049468994, + "ronpo/logit": 0.12056305259466171, + "ronpo/residual": 0.10401593148708344, + "ronpo/residual_abs": 0.1900263875722885, + "ronpo/target": 0.016547124832868576, + "ronpo/target_abs": 0.08112061023712158, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.2035134732723236, + "drift/rejected_abs": 0.1420602947473526, + "epoch": 0.1907032181168057, + "grad_norm": 147.0, + "learning_rate": 2.278066927336698e-08, + "logits/chosen": -2.131030797958374, + "logits/rejected": -2.104788303375244, + "logps/chosen": -0.29952484369277954, + "logps/rejected": -0.320384681224823, + "loss": 0.45714807510375977, + "loss/core": 0.4465022087097168, + "loss/preference_sft": 0.3080054819583893, + "loss/reference_anchor": 0.2733660936355591, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0339159965515137, + "rewards/margins": 0.6155134439468384, + "rewards/rejected": 1.4184025526046753, + "ronpo/logit": 0.06155133247375488, + "ronpo/residual": 0.06355226784944534, + "ronpo/residual_abs": 0.23413407802581787, + "ronpo/target": -0.0020009521394968033, + "ronpo/target_abs": 0.06787214428186417, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.34611085057258606, + "drift/rejected_abs": 0.15783950686454773, + "epoch": 0.19229241160111243, + "grad_norm": 1.2890625, + "learning_rate": 2.2114621138140448e-08, + "logits/chosen": -2.026627540588379, + "logits/rejected": -1.951611876487732, + "logps/chosen": -0.283507376909256, + "logps/rejected": -0.27926164865493774, + "loss": 0.4765016436576843, + "loss/core": 0.45531606674194336, + "loss/preference_sft": 0.2730947434902191, + "loss/reference_anchor": 0.5779938101768494, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.452455520629883, + "rewards/margins": 1.884347915649414, + "rewards/rejected": 1.5681076049804688, + "ronpo/logit": 0.18843480944633484, + "ronpo/residual": 0.18703269958496094, + "ronpo/residual_abs": 0.2455645054578781, + "ronpo/target": 0.0014021346578374505, + "ronpo/target_abs": 0.0817280188202858, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.3367292284965515, + "drift/rejected_abs": 0.152937650680542, + "epoch": 0.19388160508541916, + "grad_norm": 95.5, + "learning_rate": 2.145435881468797e-08, + "logits/chosen": -2.018470525741577, + "logits/rejected": -1.968064308166504, + "logps/chosen": -0.3080677092075348, + "logps/rejected": -0.29313474893569946, + "loss": 0.3543425500392914, + "loss/core": 0.3362095355987549, + "loss/preference_sft": 0.31406843662261963, + "loss/reference_anchor": 0.4866800904273987, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.365661144256592, + "rewards/margins": 1.8380382061004639, + "rewards/rejected": 1.527622938156128, + "ronpo/logit": 0.18380382657051086, + "ronpo/residual": 0.18959875404834747, + "ronpo/residual_abs": 0.23765261471271515, + "ronpo/target": -0.005794919561594725, + "ronpo/target_abs": 0.08199281990528107, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.25560295581817627, + "drift/rejected_abs": 0.11357559263706207, + "epoch": 0.19547079856972585, + "grad_norm": 125.0, + "learning_rate": 2.0800130600666648e-08, + "logits/chosen": -2.1990818977355957, + "logits/rejected": -2.159752368927002, + "logps/chosen": -0.28720757365226746, + "logps/rejected": -0.28762027621269226, + "loss": 0.293748140335083, + "loss/core": 0.28089436888694763, + "loss/preference_sft": 0.293092280626297, + "loss/reference_anchor": 0.33794093132019043, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5560295581817627, + "rewards/margins": 1.4323694705963135, + "rewards/rejected": 1.1236598491668701, + "ronpo/logit": 0.1432369500398636, + "ronpo/residual": 0.14470703899860382, + "ronpo/residual_abs": 0.18086595833301544, + "ronpo/target": -0.0014700808096677065, + "ronpo/target_abs": 0.06537780165672302, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.22184021770954132, + "drift/rejected_abs": 0.1088438630104065, + "epoch": 0.19705999205403257, + "grad_norm": 13.875, + "learning_rate": 2.0152182524551813e-08, + "logits/chosen": -2.0863685607910156, + "logits/rejected": -2.0217607021331787, + "logps/chosen": -0.29196107387542725, + "logps/rejected": -0.2876456379890442, + "loss": 0.10439170897006989, + "loss/core": 0.0983448475599289, + "loss/preference_sft": 0.2921106219291687, + "loss/reference_anchor": 0.1435559093952179, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2182328701019287, + "rewards/margins": 1.1298673152923584, + "rewards/rejected": 1.0883656740188599, + "ronpo/logit": 0.1129867285490036, + "ronpo/residual": 0.11576826870441437, + "ronpo/residual_abs": 0.16257163882255554, + "ronpo/target": -0.0027815259527415037, + "ronpo/target_abs": 0.06601886451244354, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.24640531837940216, + "drift/rejected_abs": 0.09290675818920135, + "epoch": 0.1986491855383393, + "grad_norm": 45.25, + "learning_rate": 1.951075825311578e-08, + "logits/chosen": -2.0369296073913574, + "logits/rejected": -2.0407700538635254, + "logps/chosen": -0.2959621846675873, + "logps/rejected": -0.29841041564941406, + "loss": 0.3487691283226013, + "loss/core": 0.33975881338119507, + "loss/preference_sft": 0.2982815206050873, + "loss/reference_anchor": 0.22760829329490662, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4635446071624756, + "rewards/margins": 1.5408732891082764, + "rewards/rejected": 0.9226716160774231, + "ronpo/logit": 0.15408733487129211, + "ronpo/residual": 0.17024736106395721, + "ronpo/residual_abs": 0.2152116596698761, + "ronpo/target": -0.016160037368535995, + "ronpo/target_abs": 0.07987652719020844, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.23432044684886932, + "drift/rejected_abs": 0.08939666301012039, + "epoch": 0.20023837902264602, + "grad_norm": 2.875, + "learning_rate": 1.887609899979465e-08, + "logits/chosen": -2.3256447315216064, + "logits/rejected": -2.416283130645752, + "logps/chosen": -0.2670096158981323, + "logps/rejected": -0.2636256814002991, + "loss": 0.3040315806865692, + "loss/core": 0.29226741194725037, + "loss/preference_sft": 0.2660647928714752, + "loss/reference_anchor": 0.30951163172721863, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.3428587913513184, + "rewards/margins": 1.4511168003082275, + "rewards/rejected": 0.891741931438446, + "ronpo/logit": 0.14511170983314514, + "ronpo/residual": 0.1341066062450409, + "ronpo/residual_abs": 0.19294190406799316, + "ronpo/target": 0.011005090549588203, + "ronpo/target_abs": 0.06993237882852554, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.2895382344722748, + "drift/rejected_abs": 0.10728726536035538, + "epoch": 0.2018275725069527, + "grad_norm": 16.375, + "learning_rate": 1.824844343397797e-08, + "logits/chosen": -2.1119954586029053, + "logits/rejected": -2.150533676147461, + "logps/chosen": -0.2655460238456726, + "logps/rejected": -0.2733895182609558, + "loss": 0.38659775257110596, + "loss/core": 0.37493616342544556, + "loss/preference_sft": 0.2679237723350525, + "loss/reference_anchor": 0.30639657378196716, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.8941664695739746, + "rewards/margins": 1.8319135904312134, + "rewards/rejected": 1.0622529983520508, + "ronpo/logit": 0.18319137394428253, + "ronpo/residual": 0.1691356599330902, + "ronpo/residual_abs": 0.2294747531414032, + "ronpo/target": 0.014055708423256874, + "ronpo/target_abs": 0.07849180698394775, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.32430991530418396, + "drift/rejected_abs": 0.09809484332799911, + "epoch": 0.20341676599125944, + "grad_norm": 1.703125, + "learning_rate": 1.762802759125482e-08, + "logits/chosen": -2.017073154449463, + "logits/rejected": -2.0082268714904785, + "logps/chosen": -0.31356340646743774, + "logps/rejected": -0.31727951765060425, + "loss": 0.7281572818756104, + "loss/core": 0.707539439201355, + "loss/preference_sft": 0.3098931312561035, + "loss/reference_anchor": 0.5580906867980957, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2425026893615723, + "rewards/margins": 2.297203779220581, + "rewards/rejected": 0.9452990293502808, + "ronpo/logit": 0.22972038388252258, + "ronpo/residual": 0.2376583367586136, + "ronpo/residual_abs": 0.27416878938674927, + "ronpo/target": -0.007937933318316936, + "ronpo/target_abs": 0.07765815407037735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.26535215973854065, + "drift/rejected_abs": 0.11467768251895905, + "epoch": 0.20500595947556616, + "grad_norm": 7.9375, + "learning_rate": 1.7015084784650487e-08, + "logits/chosen": -2.123807430267334, + "logits/rejected": -2.129242181777954, + "logps/chosen": -0.3087415099143982, + "logps/rejected": -0.2945675551891327, + "loss": 0.3146616816520691, + "loss/core": 0.30256468057632446, + "loss/preference_sft": 0.3151400685310364, + "loss/reference_anchor": 0.31411463022232056, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.6520421504974365, + "rewards/margins": 1.5085344314575195, + "rewards/rejected": 1.143507957458496, + "ronpo/logit": 0.15085344016551971, + "ronpo/residual": 0.14317695796489716, + "ronpo/residual_abs": 0.20519272983074188, + "ronpo/target": 0.007676492445170879, + "ronpo/target_abs": 0.08047398179769516, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.15140828490257263, + "drift/rejected_abs": 0.09236256033182144, + "epoch": 0.20659515295987285, + "grad_norm": 1.84375, + "learning_rate": 1.6409845516887003e-08, + "logits/chosen": -2.207329273223877, + "logits/rejected": -2.135587215423584, + "logps/chosen": -0.3012005686759949, + "logps/rejected": -0.29473382234573364, + "loss": 0.0722493976354599, + "loss/core": 0.06875967979431152, + "loss/preference_sft": 0.3042813539505005, + "loss/reference_anchor": 0.06927807629108429, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5135061740875244, + "rewards/margins": 0.6719859838485718, + "rewards/rejected": 0.8415201902389526, + "ronpo/logit": 0.06719859689474106, + "ronpo/residual": 0.0728544145822525, + "ronpo/residual_abs": 0.12853005528450012, + "ronpo/target": -0.005655817687511444, + "ronpo/target_abs": 0.07488958537578583, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.20458777248859406, + "drift/rejected_abs": 0.06509005278348923, + "epoch": 0.20818434644417957, + "grad_norm": 172.0, + "learning_rate": 1.5812537393700506e-08, + "logits/chosen": -2.2815442085266113, + "logits/rejected": -2.287656784057617, + "logps/chosen": -0.28355464339256287, + "logps/rejected": -0.29816484451293945, + "loss": 0.39056888222694397, + "loss/core": 0.3819330930709839, + "loss/preference_sft": 0.2903548777103424, + "loss/reference_anchor": 0.2177002727985382, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0455703735351562, + "rewards/margins": 1.397626519203186, + "rewards/rejected": 0.6479436755180359, + "ronpo/logit": 0.13976263999938965, + "ronpo/residual": 0.16489243507385254, + "ronpo/residual_abs": 0.20165614783763885, + "ronpo/target": -0.025129785761237144, + "ronpo/target_abs": 0.08682605624198914, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.17608028650283813, + "drift/rejected_abs": 0.07316999137401581, + "epoch": 0.2097735399284863, + "grad_norm": 8.0, + "learning_rate": 1.522338503824803e-08, + "logits/chosen": -2.2558116912841797, + "logits/rejected": -2.2970988750457764, + "logps/chosen": -0.29492825269699097, + "logps/rejected": -0.29021143913269043, + "loss": 0.12977449595928192, + "loss/core": 0.1249893456697464, + "loss/preference_sft": 0.29489555954933167, + "loss/reference_anchor": 0.10722937434911728, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.7590672969818115, + "rewards/margins": 1.0307245254516602, + "rewards/rejected": 0.7283426523208618, + "ronpo/logit": 0.10307244956493378, + "ronpo/residual": 0.1016714796423912, + "ronpo/residual_abs": 0.14400455355644226, + "ronpo/target": 0.0014009776059538126, + "ronpo/target_abs": 0.06940744817256927, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.2352174073457718, + "drift/rejected_abs": 0.13656437397003174, + "epoch": 0.21136273341279302, + "grad_norm": 0.79296875, + "learning_rate": 1.464261000663589e-08, + "logits/chosen": -2.079434871673584, + "logits/rejected": -2.024263858795166, + "logps/chosen": -0.2820441424846649, + "logps/rejected": -0.2857016921043396, + "loss": 0.23443233966827393, + "loss/core": 0.2236245572566986, + "loss/preference_sft": 0.2865526080131531, + "loss/reference_anchor": 0.28013843297958374, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.349414825439453, + "rewards/margins": 0.9838077425956726, + "rewards/rejected": 1.3656069040298462, + "ronpo/logit": 0.09838077425956726, + "ronpo/residual": 0.11156570911407471, + "ronpo/residual_abs": 0.17216257750988007, + "ronpo/target": -0.013184946961700916, + "ronpo/target_abs": 0.07473040372133255, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.20989990234375, + "drift/rejected_abs": 0.07849745452404022, + "epoch": 0.2129519268970997, + "grad_norm": 5.40625, + "learning_rate": 1.40704307046015e-08, + "logits/chosen": -2.347775459289551, + "logits/rejected": -2.3573763370513916, + "logps/chosen": -0.2676052153110504, + "logps/rejected": -0.27207422256469727, + "loss": 0.21361777186393738, + "loss/core": 0.20699867606163025, + "loss/preference_sft": 0.26641157269477844, + "loss/reference_anchor": 0.16247610747814178, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0982730388641357, + "rewards/margins": 1.3146512508392334, + "rewards/rejected": 0.7836218476295471, + "ronpo/logit": 0.1314651221036911, + "ronpo/residual": 0.16295532882213593, + "ronpo/residual_abs": 0.18369440734386444, + "ronpo/target": -0.03149022161960602, + "ronpo/target_abs": 0.07089079916477203, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.4036272466182709, + "drift/rejected_abs": 0.08043356984853745, + "epoch": 0.21454112038140644, + "grad_norm": 2.21875, + "learning_rate": 1.3507062305379941e-08, + "logits/chosen": -1.9893615245819092, + "logits/rejected": -2.1025524139404297, + "logps/chosen": -0.2764974534511566, + "logps/rejected": -0.27073296904563904, + "loss": 1.4822903871536255, + "loss/core": 1.4529348611831665, + "loss/preference_sft": 0.27298080921173096, + "loss/reference_anchor": 0.8114289045333862, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.034860134124756, + "rewards/margins": 3.2362799644470215, + "rewards/rejected": 0.7985801100730896, + "ronpo/logit": 0.3236280679702759, + "ronpo/residual": 0.3155183792114258, + "ronpo/residual_abs": 0.37980663776397705, + "ronpo/target": 0.008109642192721367, + "ronpo/target_abs": 0.08955520391464233, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.19523149728775024, + "drift/rejected_abs": 0.11687342822551727, + "epoch": 0.21613031386571316, + "grad_norm": 2.484375, + "learning_rate": 1.2952716668786109e-08, + "logits/chosen": -1.9566532373428345, + "logits/rejected": -1.9517863988876343, + "logps/chosen": -0.28758880496025085, + "logps/rejected": -0.28461915254592896, + "loss": 0.066526398062706, + "loss/core": 0.05917876213788986, + "loss/preference_sft": 0.28697067499160767, + "loss/reference_anchor": 0.18123537302017212, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.950860619544983, + "rewards/margins": 0.7826768159866333, + "rewards/rejected": 1.1681835651397705, + "ronpo/logit": 0.07826767861843109, + "ronpo/residual": 0.07248474657535553, + "ronpo/residual_abs": 0.13983985781669617, + "ronpo/target": 0.0057829380966722965, + "ronpo/target_abs": 0.08430159091949463, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.18238039314746857, + "drift/rejected_abs": 0.09428268671035767, + "epoch": 0.21771950735001985, + "grad_norm": 3.3125, + "learning_rate": 1.2407602261542828e-08, + "logits/chosen": -1.9722627401351929, + "logits/rejected": -2.0617997646331787, + "logps/chosen": -0.2924751937389374, + "logps/rejected": -0.29996949434280396, + "loss": 0.054906345903873444, + "loss/core": 0.0511222667992115, + "loss/preference_sft": 0.30433911085128784, + "loss/reference_anchor": 0.07768283784389496, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8226171731948853, + "rewards/margins": 0.8817318677902222, + "rewards/rejected": 0.9408853650093079, + "ronpo/logit": 0.08817318081855774, + "ronpo/residual": 0.08928640186786652, + "ronpo/residual_abs": 0.133590966463089, + "ronpo/target": -0.0011132096406072378, + "ronpo/target_abs": 0.07680922746658325, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.45133644342422485, + "drift/rejected_abs": 0.18786491453647614, + "epoch": 0.21930870083432658, + "grad_norm": 434.0, + "learning_rate": 1.1871924078885106e-08, + "logits/chosen": -1.929914116859436, + "logits/rejected": -1.9743080139160156, + "logps/chosen": -0.2774960696697235, + "logps/rejected": -0.26925212144851685, + "loss": 0.8445002436637878, + "loss/core": 0.8047998547554016, + "loss/preference_sft": 0.2817479968070984, + "loss/reference_anchor": 1.1061266660690308, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.513057708740234, + "rewards/margins": 2.6355862617492676, + "rewards/rejected": 1.8774709701538086, + "ronpo/logit": 0.26355868577957153, + "ronpo/residual": 0.26055908203125, + "ronpo/residual_abs": 0.3207409679889679, + "ronpo/target": 0.002999568125233054, + "ronpo/target_abs": 0.0797749012708664, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.1836492121219635, + "drift/rejected_abs": 0.10975848138332367, + "epoch": 0.2208978943186333, + "grad_norm": 12.375, + "learning_rate": 1.1345883567469826e-08, + "logits/chosen": -2.1611971855163574, + "logits/rejected": -2.147216558456421, + "logps/chosen": -0.2980535626411438, + "logps/rejected": -0.30087488889694214, + "loss": 0.11188419908285141, + "loss/core": 0.10657135397195816, + "loss/preference_sft": 0.2982610762119293, + "loss/reference_anchor": 0.1219698041677475, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8356151580810547, + "rewards/margins": 0.7388324737548828, + "rewards/rejected": 1.0967828035354614, + "ronpo/logit": 0.07388325035572052, + "ronpo/residual": 0.06399385631084442, + "ronpo/residual_abs": 0.16593526303768158, + "ronpo/target": 0.00988937821239233, + "ronpo/target_abs": 0.07012668997049332, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.2231002300977707, + "drift/rejected_abs": 0.13982199132442474, + "epoch": 0.22248708780294002, + "grad_norm": 6.5625, + "learning_rate": 1.0829678549619859e-08, + "logits/chosen": -2.218724489212036, + "logits/rejected": -2.1778321266174316, + "logps/chosen": -0.32158035039901733, + "logps/rejected": -0.3115960955619812, + "loss": 0.43686145544052124, + "loss/core": 0.42180243134498596, + "loss/preference_sft": 0.31540364027023315, + "loss/reference_anchor": 0.39871746301651, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2305080890655518, + "rewards/margins": 0.8341946601867676, + "rewards/rejected": 1.3963135480880737, + "ronpo/logit": 0.08341946452856064, + "ronpo/residual": 0.08280835300683975, + "ronpo/residual_abs": 0.19750329852104187, + "ronpo/target": 0.0006111121037974954, + "ronpo/target_abs": 0.0668053850531578, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.1843855381011963, + "drift/rejected_abs": 0.08389295637607574, + "epoch": 0.22407628128724671, + "grad_norm": 3.328125, + "learning_rate": 1.0323503148931167e-08, + "logits/chosen": -2.261746644973755, + "logits/rejected": -2.3672595024108887, + "logps/chosen": -0.26710596680641174, + "logps/rejected": -0.27074798941612244, + "loss": 0.08147259801626205, + "loss/core": 0.07696676254272461, + "loss/preference_sft": 0.27835264801979065, + "loss/reference_anchor": 0.10090293735265732, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8432375192642212, + "rewards/margins": 1.0078269243240356, + "rewards/rejected": 0.8354104161262512, + "ronpo/logit": 0.10078269243240356, + "ronpo/residual": 0.10459145158529282, + "ronpo/residual_abs": 0.15122047066688538, + "ronpo/target": -0.0038087614811956882, + "ronpo/target_abs": 0.08357212692499161, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.2887698709964752, + "drift/rejected_abs": 0.10348562896251678, + "epoch": 0.22566547477155344, + "grad_norm": 107.5, + "learning_rate": 9.827547717270884e-09, + "logits/chosen": -2.164956569671631, + "logits/rejected": -2.177293300628662, + "logps/chosen": -0.29969245195388794, + "logps/rejected": -0.29687872529029846, + "loss": 0.6983797550201416, + "loss/core": 0.6833909749984741, + "loss/preference_sft": 0.2986063063144684, + "loss/reference_anchor": 0.3983899652957916, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.8871381282806396, + "rewards/margins": 1.9205951690673828, + "rewards/rejected": 0.9665428400039673, + "ronpo/logit": 0.19205954670906067, + "ronpo/residual": 0.18855328857898712, + "ronpo/residual_abs": 0.2506033778190613, + "ronpo/target": 0.003506250213831663, + "ronpo/target_abs": 0.06901577860116959, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.1665627658367157, + "drift/rejected_abs": 0.0922744944691658, + "epoch": 0.22725466825586016, + "grad_norm": 1.1171875, + "learning_rate": 9.34199876319378e-09, + "logits/chosen": -2.0484366416931152, + "logits/rejected": -1.9690176248550415, + "logps/chosen": -0.3025513291358948, + "logps/rejected": -0.30018630623817444, + "loss": 0.07143449038267136, + "loss/core": 0.06748920679092407, + "loss/preference_sft": 0.3050025403499603, + "loss/reference_anchor": 0.08222213387489319, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6645787954330444, + "rewards/margins": 0.7441354990005493, + "rewards/rejected": 0.9204433560371399, + "ronpo/logit": 0.07441355288028717, + "ronpo/residual": 0.06392879039049149, + "ronpo/residual_abs": 0.13627013564109802, + "ronpo/target": 0.010484759695827961, + "ronpo/target_abs": 0.0739731639623642, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.3448890745639801, + "drift/rejected_abs": 0.14354726672172546, + "epoch": 0.22884386174016685, + "grad_norm": 482.0, + "learning_rate": 8.867038881803992e-09, + "logits/chosen": -1.9485433101654053, + "logits/rejected": -2.0286898612976074, + "logps/chosen": -0.2920309901237488, + "logps/rejected": -0.2847922146320343, + "loss": 0.5865887403488159, + "loss/core": 0.5658643841743469, + "loss/preference_sft": 0.2796561121940613, + "loss/reference_anchor": 0.5641579031944275, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4488911628723145, + "rewards/margins": 2.0442280769348145, + "rewards/rejected": 1.4046627283096313, + "ronpo/logit": 0.20442280173301697, + "ronpo/residual": 0.18873873353004456, + "ronpo/residual_abs": 0.24582239985466003, + "ronpo/target": 0.015684079378843307, + "ronpo/target_abs": 0.07453683018684387, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.2959468960762024, + "drift/rejected_abs": 0.06901375949382782, + "epoch": 0.23043305522447358, + "grad_norm": 4.0625, + "learning_rate": 8.402846686088465e-09, + "logits/chosen": -2.08479642868042, + "logits/rejected": -2.0257925987243652, + "logps/chosen": -0.29016560316085815, + "logps/rejected": -0.2911625802516937, + "loss": 1.0842726230621338, + "loss/core": 1.0630663633346558, + "loss/preference_sft": 0.2913043797016144, + "loss/reference_anchor": 0.5767573118209839, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.959277629852295, + "rewards/margins": 2.2727534770965576, + "rewards/rejected": 0.6865240931510925, + "ronpo/logit": 0.22727534174919128, + "ronpo/residual": 0.20963330566883087, + "ronpo/residual_abs": 0.27996546030044556, + "ronpo/target": 0.017642047256231308, + "ronpo/target_abs": 0.08470751345157623, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.31818053126335144, + "drift/rejected_abs": 0.09311111271381378, + "epoch": 0.2320222487087803, + "grad_norm": 2656.0, + "learning_rate": 7.949596739747928e-09, + "logits/chosen": -2.1799933910369873, + "logits/rejected": -2.214536190032959, + "logps/chosen": -0.2960435748100281, + "logps/rejected": -0.2987004816532135, + "loss": 1.0148961544036865, + "loss/core": 0.9916189908981323, + "loss/preference_sft": 0.30723002552986145, + "loss/reference_anchor": 0.6343376040458679, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.18082332611084, + "rewards/margins": 2.2561564445495605, + "rewards/rejected": 0.9246670007705688, + "ronpo/logit": 0.22561565041542053, + "ronpo/residual": 0.211147278547287, + "ronpo/residual_abs": 0.27076631784439087, + "ronpo/target": 0.014468356966972351, + "ronpo/target_abs": 0.07857581973075867, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.41159167885780334, + "drift/rejected_abs": 0.15270794928073883, + "epoch": 0.23361144219308702, + "grad_norm": 9.9375, + "learning_rate": 7.507459491550653e-09, + "logits/chosen": -1.872210144996643, + "logits/rejected": -1.9234001636505127, + "logps/chosen": -0.31087052822113037, + "logps/rejected": -0.28266483545303345, + "loss": 1.28376042842865, + "loss/core": 1.2525687217712402, + "loss/preference_sft": 0.30879443883895874, + "loss/reference_anchor": 0.8603096008300781, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.114415645599365, + "rewards/margins": 2.59476900100708, + "rewards/rejected": 1.5196462869644165, + "ronpo/logit": 0.259476900100708, + "ronpo/residual": 0.2532740831375122, + "ronpo/residual_abs": 0.36283624172210693, + "ronpo/target": 0.006202817894518375, + "ronpo/target_abs": 0.09357239305973053, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.40805894136428833, + "drift/rejected_abs": 0.17579728364944458, + "epoch": 0.23520063567739372, + "grad_norm": 6.1875, + "learning_rate": 7.076601211233599e-09, + "logits/chosen": -2.061128854751587, + "logits/rejected": -2.0528006553649902, + "logps/chosen": -0.27439576387405396, + "logps/rejected": -0.2714352607727051, + "loss": 0.8380556106567383, + "loss/core": 0.8007208704948425, + "loss/preference_sft": 0.2815498411655426, + "loss/reference_anchor": 1.0385510921478271, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.079874515533447, + "rewards/margins": 2.3227553367614746, + "rewards/rejected": 1.7571195363998413, + "ronpo/logit": 0.23227553069591522, + "ronpo/residual": 0.21789321303367615, + "ronpo/residual_abs": 0.29552918672561646, + "ronpo/target": 0.01438230276107788, + "ronpo/target_abs": 0.08076776564121246, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.2793598175048828, + "drift/rejected_abs": 0.1420426368713379, + "epoch": 0.23678982916170044, + "grad_norm": 4.1875, + "learning_rate": 6.6571839269751625e-09, + "logits/chosen": -1.9656693935394287, + "logits/rejected": -1.9407997131347656, + "logps/chosen": -0.310263454914093, + "logps/rejected": -0.2949405312538147, + "loss": 0.4082837998867035, + "loss/core": 0.39522331953048706, + "loss/preference_sft": 0.302360862493515, + "loss/reference_anchor": 0.34292274713516235, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7910044193267822, + "rewards/margins": 1.4027537107467651, + "rewards/rejected": 1.3882505893707275, + "ronpo/logit": 0.14027537405490875, + "ronpo/residual": 0.14826272428035736, + "ronpo/residual_abs": 0.2111869603395462, + "ronpo/target": -0.007987345568835735, + "ronpo/target_abs": 0.06692041456699371, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.4917718470096588, + "drift/rejected_abs": 0.09017399698495865, + "epoch": 0.23837902264600716, + "grad_norm": 25.875, + "learning_rate": 6.249365364463068e-09, + "logits/chosen": -1.8745949268341064, + "logits/rejected": -2.0258686542510986, + "logps/chosen": -0.32771915197372437, + "logps/rejected": -0.3020108938217163, + "loss": 1.669013261795044, + "loss/core": 1.633209228515625, + "loss/preference_sft": 0.3086247444152832, + "loss/reference_anchor": 0.9921077489852905, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.917025566101074, + "rewards/margins": 4.016125679016113, + "rewards/rejected": 0.9008998870849609, + "ronpo/logit": 0.4016125798225403, + "ronpo/residual": 0.39909854531288147, + "ronpo/residual_abs": 0.4497567117214203, + "ronpo/target": 0.002514071064069867, + "ronpo/target_abs": 0.08284027129411697, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.5578740835189819, + "drift/rejected_abs": 0.25196394324302673, + "epoch": 0.23996821613031386, + "grad_norm": 176.0, + "learning_rate": 5.853298887580098e-09, + "logits/chosen": -1.7803856134414673, + "logits/rejected": -1.7086734771728516, + "logps/chosen": -0.26622751355171204, + "logps/rejected": -0.2793108820915222, + "loss": 1.8586667776107788, + "loss/core": 1.813750982284546, + "loss/preference_sft": 0.258129745721817, + "loss/reference_anchor": 1.257493019104004, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 5.57759952545166, + "rewards/margins": 3.06099009513855, + "rewards/rejected": 2.5166096687316895, + "ronpo/logit": 0.3060990273952484, + "ronpo/residual": 0.29286080598831177, + "ronpo/residual_abs": 0.49150973558425903, + "ronpo/target": 0.013238211162388325, + "ronpo/target_abs": 0.07374797761440277, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.2570843994617462, + "drift/rejected_abs": 0.10265134274959564, + "epoch": 0.24155740961462058, + "grad_norm": 1.515625, + "learning_rate": 5.4691334407302935e-09, + "logits/chosen": -2.2019875049591064, + "logits/rejected": -2.2530570030212402, + "logps/chosen": -0.26051491498947144, + "logps/rejected": -0.2609294652938843, + "loss": 0.4085308611392975, + "loss/core": 0.3968563675880432, + "loss/preference_sft": 0.27111831307411194, + "loss/reference_anchor": 0.3064435124397278, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5702319145202637, + "rewards/margins": 1.548225998878479, + "rewards/rejected": 1.0220061540603638, + "ronpo/logit": 0.15482258796691895, + "ronpo/residual": 0.14783695340156555, + "ronpo/residual_abs": 0.20230190455913544, + "ronpo/target": 0.0069856420159339905, + "ronpo/target_abs": 0.0757329910993576, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.16770626604557037, + "drift/rejected_abs": 0.09157443791627884, + "epoch": 0.2431466030989273, + "grad_norm": 1.203125, + "learning_rate": 5.097013492826982e-09, + "logits/chosen": -2.3989851474761963, + "logits/rejected": -2.3377420902252197, + "logps/chosen": -0.2859249711036682, + "logps/rejected": -0.28967946767807007, + "loss": 0.04619941860437393, + "loss/core": 0.041453104466199875, + "loss/preference_sft": 0.28654763102531433, + "loss/reference_anchor": 0.10695407539606094, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6764370203018188, + "rewards/margins": 0.7609391212463379, + "rewards/rejected": 0.9154976606369019, + "ronpo/logit": 0.07609390467405319, + "ronpo/residual": 0.09639062732458115, + "ronpo/residual_abs": 0.12980125844478607, + "ronpo/target": -0.020296704024076462, + "ronpo/target_abs": 0.06380324065685272, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.33088406920433044, + "drift/rejected_abs": 0.10067299753427505, + "epoch": 0.24473579658323402, + "grad_norm": 49.5, + "learning_rate": 4.737078982963998e-09, + "logits/chosen": -1.9620716571807861, + "logits/rejected": -2.093608856201172, + "logps/chosen": -0.2551668882369995, + "logps/rejected": -0.27200862765312195, + "loss": 0.6154618263244629, + "loss/core": 0.6015649437904358, + "loss/preference_sft": 0.26770323514938354, + "loss/reference_anchor": 0.3702848255634308, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.3076298236846924, + "rewards/margins": 2.3778507709503174, + "rewards/rejected": 0.929779052734375, + "ronpo/logit": 0.23778507113456726, + "ronpo/residual": 0.2486831247806549, + "ronpo/residual_abs": 0.29293423891067505, + "ronpo/target": -0.010898038744926453, + "ronpo/target_abs": 0.08667393028736115, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.34422239661216736, + "drift/rejected_abs": 0.14332148432731628, + "epoch": 0.24632499006754072, + "grad_norm": 1.125, + "learning_rate": 4.389465267790238e-09, + "logits/chosen": -1.8292220830917358, + "logits/rejected": -1.8172327280044556, + "logps/chosen": -0.2949088513851166, + "logps/rejected": -0.2962006628513336, + "loss": 0.5463058948516846, + "loss/core": 0.5306318402290344, + "loss/preference_sft": 0.29769080877304077, + "loss/reference_anchor": 0.41806188225746155, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.440873384475708, + "rewards/margins": 2.0447137355804443, + "rewards/rejected": 1.3961597681045532, + "ronpo/logit": 0.2044713944196701, + "ronpo/residual": 0.21325626969337463, + "ronpo/residual_abs": 0.2596508264541626, + "ronpo/target": -0.008784900419414043, + "ronpo/target_abs": 0.08347609639167786, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.3502070903778076, + "drift/rejected_abs": 0.15574514865875244, + "epoch": 0.24791418355184744, + "grad_norm": 4.03125, + "learning_rate": 4.05430307060764e-09, + "logits/chosen": -2.158703565597534, + "logits/rejected": -2.1818182468414307, + "logps/chosen": -0.2700011134147644, + "logps/rejected": -0.2787485718727112, + "loss": 0.3507295846939087, + "loss/core": 0.33437737822532654, + "loss/preference_sft": 0.27247267961502075, + "loss/reference_anchor": 0.43995794653892517, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.501782178878784, + "rewards/margins": 1.9450876712799072, + "rewards/rejected": 1.556694507598877, + "ronpo/logit": 0.19450877606868744, + "ronpo/residual": 0.18948352336883545, + "ronpo/residual_abs": 0.27817171812057495, + "ronpo/target": 0.0050252461805939674, + "ronpo/target_abs": 0.0843028575181961, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.47840428352355957, + "drift/rejected_abs": 0.1084831953048706, + "epoch": 0.24950337703615416, + "grad_norm": 4.3125, + "learning_rate": 3.731718432211417e-09, + "logits/chosen": -2.1047401428222656, + "logits/rejected": -2.21636962890625, + "logps/chosen": -0.2704011797904968, + "logps/rejected": -0.26196351647377014, + "loss": 1.810654878616333, + "loss/core": 1.7768656015396118, + "loss/preference_sft": 0.26972612738609314, + "loss/reference_anchor": 0.9384282827377319, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.778249263763428, + "rewards/margins": 3.708155870437622, + "rewards/rejected": 1.0700935125350952, + "ronpo/logit": 0.37081560492515564, + "ronpo/residual": 0.3735385537147522, + "ronpo/residual_abs": 0.4183095097541809, + "ronpo/target": -0.0027229245752096176, + "ronpo/target_abs": 0.08420609682798386, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.3598479926586151, + "drift/rejected_abs": 0.12107064574956894, + "epoch": 0.25109257052046086, + "grad_norm": 191.0, + "learning_rate": 3.4218326634913406e-09, + "logits/chosen": -2.0067882537841797, + "logits/rejected": -2.0267016887664795, + "logps/chosen": -0.31197962164878845, + "logps/rejected": -0.29012903571128845, + "loss": 0.4827456474304199, + "loss/core": 0.46548813581466675, + "loss/preference_sft": 0.307406485080719, + "loss/reference_anchor": 0.46233057975769043, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.597403049468994, + "rewards/margins": 2.416919231414795, + "rewards/rejected": 1.1804835796356201, + "ronpo/logit": 0.2416919767856598, + "ronpo/residual": 0.2469751387834549, + "ronpo/residual_abs": 0.3075826168060303, + "ronpo/target": -0.005283153150230646, + "ronpo/target_abs": 0.07869095355272293, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.24024005234241486, + "drift/rejected_abs": 0.11211339384317398, + "epoch": 0.2526817640047676, + "grad_norm": 1.140625, + "learning_rate": 3.1247622998116148e-09, + "logits/chosen": -2.133969783782959, + "logits/rejected": -2.1671390533447266, + "logps/chosen": -0.25962501764297485, + "logps/rejected": -0.25368732213974, + "loss": 0.12422094494104385, + "loss/core": 0.11874745786190033, + "loss/preference_sft": 0.2524915635585785, + "loss/reference_anchor": 0.13113582134246826, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.401646375656128, + "rewards/margins": 1.2813524007797241, + "rewards/rejected": 1.1202940940856934, + "ronpo/logit": 0.12813523411750793, + "ronpo/residual": 0.1344965100288391, + "ronpo/residual_abs": 0.19210845232009888, + "ronpo/target": -0.006361273117363453, + "ronpo/target_abs": 0.07786816358566284, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.2830193042755127, + "drift/rejected_abs": 0.15909279882907867, + "epoch": 0.2542709574890743, + "grad_norm": 16.75, + "learning_rate": 2.840619057186745e-09, + "logits/chosen": -2.169642448425293, + "logits/rejected": -2.2404894828796387, + "logps/chosen": -0.2703879475593567, + "logps/rejected": -0.2701282203197479, + "loss": 0.2124510258436203, + "loss/core": 0.1838868260383606, + "loss/preference_sft": 0.2592373192310333, + "loss/reference_anchor": 0.7901961803436279, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8296594619750977, + "rewards/margins": 1.2395825386047363, + "rewards/rejected": 1.5900771617889404, + "ronpo/logit": 0.12395823001861572, + "ronpo/residual": 0.1275547742843628, + "ronpo/residual_abs": 0.1707366406917572, + "ronpo/target": -0.003596528898924589, + "ronpo/target_abs": 0.06492110341787338, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.3190115988254547, + "drift/rejected_abs": 0.13805243372917175, + "epoch": 0.255860150973381, + "grad_norm": 217.0, + "learning_rate": 2.5695097902696582e-09, + "logits/chosen": -2.2752726078033447, + "logits/rejected": -2.237405300140381, + "logps/chosen": -0.30355507135391235, + "logps/rejected": -0.3101564049720764, + "loss": 0.8964834213256836, + "loss/core": 0.8716424703598022, + "loss/preference_sft": 0.2993467450141907, + "loss/reference_anchor": 0.6798084378242493, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1900362968444824, + "rewards/margins": 1.8883402347564697, + "rewards/rejected": 1.3016960620880127, + "ronpo/logit": 0.1888340413570404, + "ronpo/residual": 0.1919974535703659, + "ronpo/residual_abs": 0.29645854234695435, + "ronpo/target": -0.0031634189654141665, + "ronpo/target_abs": 0.07282833755016327, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.21134933829307556, + "drift/rejected_abs": 0.11000971496105194, + "epoch": 0.25744934445768775, + "grad_norm": 7.40625, + "learning_rate": 2.311536452168106e-09, + "logits/chosen": -2.0637238025665283, + "logits/rejected": -2.114058494567871, + "logps/chosen": -0.28332144021987915, + "logps/rejected": -0.28769245743751526, + "loss": 0.04527657851576805, + "loss/core": 0.04074571654200554, + "loss/preference_sft": 0.2843453884124756, + "loss/reference_anchor": 0.10101868957281113, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1122398376464844, + "rewards/margins": 1.0128422975540161, + "rewards/rejected": 1.0993976593017578, + "ronpo/logit": 0.10128422826528549, + "ronpo/residual": 0.07714606821537018, + "ronpo/residual_abs": 0.1348331868648529, + "ronpo/target": 0.02413817122578621, + "ronpo/target_abs": 0.07104022800922394, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.2593992352485657, + "drift/rejected_abs": 0.16390137374401093, + "epoch": 0.25903853794199444, + "grad_norm": 12.125, + "learning_rate": 2.0667960561041974e-09, + "logits/chosen": -2.107893228530884, + "logits/rejected": -2.0309271812438965, + "logps/chosen": -0.2672129273414612, + "logps/rejected": -0.2773855924606323, + "loss": 0.2365700900554657, + "loss/core": 0.22861385345458984, + "loss/preference_sft": 0.2656956911087036, + "loss/reference_anchor": 0.20075194537639618, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.593992233276367, + "rewards/margins": 0.9726476669311523, + "rewards/rejected": 1.6213443279266357, + "ronpo/logit": 0.09726477414369583, + "ronpo/residual": 0.08035314828157425, + "ronpo/residual_abs": 0.22539310157299042, + "ronpo/target": 0.016911622136831284, + "ronpo/target_abs": 0.08380316197872162, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.1997155249118805, + "drift/rejected_abs": 0.1135120838880539, + "epoch": 0.26062773142630113, + "grad_norm": 1.828125, + "learning_rate": 1.8353806389317424e-09, + "logits/chosen": -2.0197744369506836, + "logits/rejected": -1.9759705066680908, + "logps/chosen": -0.2759111225605011, + "logps/rejected": -0.28906577825546265, + "loss": 0.06284098327159882, + "loss/core": 0.056352388113737106, + "loss/preference_sft": 0.27423539757728577, + "loss/reference_anchor": 0.15796485543251038, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.993607521057129, + "rewards/margins": 0.8597732782363892, + "rewards/rejected": 1.1338340044021606, + "ronpo/logit": 0.08597733825445175, + "ronpo/residual": 0.08307936042547226, + "ronpo/residual_abs": 0.14249305427074432, + "ronpo/target": 0.0028979722410440445, + "ronpo/target_abs": 0.07825162261724472, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.2867492735385895, + "drift/rejected_abs": 0.18776382505893707, + "epoch": 0.2622169249106079, + "grad_norm": 67.5, + "learning_rate": 1.6173772265249478e-09, + "logits/chosen": -2.2137134075164795, + "logits/rejected": -2.161581516265869, + "logps/chosen": -0.2998419404029846, + "logps/rejected": -0.3022862374782562, + "loss": 0.3233335018157959, + "loss/core": 0.29298847913742065, + "loss/preference_sft": 0.3009437024593353, + "loss/reference_anchor": 0.8369055986404419, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8670103549957275, + "rewards/margins": 0.9923250079154968, + "rewards/rejected": 1.8746850490570068, + "ronpo/logit": 0.0992325022816658, + "ronpo/residual": 0.09889347851276398, + "ronpo/residual_abs": 0.20365075767040253, + "ronpo/target": 0.00033902880386449397, + "ronpo/target_abs": 0.07141275703907013, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.18498516082763672, + "drift/rejected_abs": 0.11326082795858383, + "epoch": 0.2638061183949146, + "grad_norm": 35.5, + "learning_rate": 1.4128678010515385e-09, + "logits/chosen": -2.2336018085479736, + "logits/rejected": -2.234981060028076, + "logps/chosen": -0.2774164378643036, + "logps/rejected": -0.2730526328086853, + "loss": 0.13492383062839508, + "loss/core": 0.12916125357151031, + "loss/preference_sft": 0.27852898836135864, + "loss/reference_anchor": 0.13679227232933044, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8494479656219482, + "rewards/margins": 0.7175174951553345, + "rewards/rejected": 1.1319307088851929, + "ronpo/logit": 0.07175175100564957, + "ronpo/residual": 0.05915927141904831, + "ronpo/residual_abs": 0.17424622178077698, + "ronpo/target": 0.012592477723956108, + "ronpo/target_abs": 0.0779508426785469, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.4061504006385803, + "drift/rejected_abs": 0.16050119698047638, + "epoch": 0.2653953118792213, + "grad_norm": 17.25, + "learning_rate": 1.2219292701426076e-09, + "logits/chosen": -2.190483808517456, + "logits/rejected": -2.155860424041748, + "logps/chosen": -0.24286535382270813, + "logps/rejected": -0.24661651253700256, + "loss": 1.0600229501724243, + "loss/core": 1.0351521968841553, + "loss/preference_sft": 0.24154765903949738, + "loss/reference_anchor": 0.686432957649231, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 4.060586452484131, + "rewards/margins": 2.4555745124816895, + "rewards/rejected": 1.6050119400024414, + "ronpo/logit": 0.24555747210979462, + "ronpo/residual": 0.233054518699646, + "ronpo/residual_abs": 0.3451952636241913, + "ronpo/target": 0.012502968311309814, + "ronpo/target_abs": 0.07277142256498337, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.28720027208328247, + "drift/rejected_abs": 0.13702043890953064, + "epoch": 0.266984505363528, + "grad_norm": 1.0546875, + "learning_rate": 1.0446334379708085e-09, + "logits/chosen": -2.3651621341705322, + "logits/rejected": -2.4263358116149902, + "logps/chosen": -0.26630455255508423, + "logps/rejected": -0.2645034193992615, + "loss": 0.3639865517616272, + "loss/core": 0.34910276532173157, + "loss/preference_sft": 0.2732158899307251, + "loss/reference_anchor": 0.39793118834495544, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.8720028400421143, + "rewards/margins": 1.5028787851333618, + "rewards/rejected": 1.369124174118042, + "ronpo/logit": 0.15028786659240723, + "ronpo/residual": 0.15997345745563507, + "ronpo/residual_abs": 0.20985810458660126, + "ronpo/target": -0.009685601107776165, + "ronpo/target_abs": 0.08229205757379532, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.29841169714927673, + "drift/rejected_abs": 0.18531523644924164, + "epoch": 0.2685736988478347, + "grad_norm": 6.65625, + "learning_rate": 8.810469782477639e-10, + "logits/chosen": -2.1420059204101562, + "logits/rejected": -2.135344982147217, + "logps/chosen": -0.24753496050834656, + "logps/rejected": -0.23413236439228058, + "loss": 0.0860786885023117, + "loss/core": 0.07448365539312363, + "loss/preference_sft": 0.24143639206886292, + "loss/reference_anchor": 0.30714288353919983, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.9827346801757812, + "rewards/margins": 1.1305488348007202, + "rewards/rejected": 1.8521859645843506, + "ronpo/logit": 0.11305488646030426, + "ronpo/residual": 0.10625892877578735, + "ronpo/residual_abs": 0.16595976054668427, + "ronpo/target": 0.006795956287533045, + "ronpo/target_abs": 0.08061005175113678, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.41207045316696167, + "drift/rejected_abs": 0.1471131145954132, + "epoch": 0.2701628923321414, + "grad_norm": 82.5, + "learning_rate": 7.31231409150776e-10, + "logits/chosen": -1.953844428062439, + "logits/rejected": -1.8874391317367554, + "logps/chosen": -0.2713695466518402, + "logps/rejected": -0.2831560969352722, + "loss": 0.9032095670700073, + "loss/core": 0.8764148950576782, + "loss/preference_sft": 0.2667026221752167, + "loss/reference_anchor": 0.7388928532600403, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.118669509887695, + "rewards/margins": 2.6498663425445557, + "rewards/rejected": 1.4688036441802979, + "ronpo/logit": 0.26498663425445557, + "ronpo/residual": 0.2674812972545624, + "ronpo/residual_abs": 0.32319194078445435, + "ronpo/target": -0.0024946490302681923, + "ronpo/target_abs": 0.07483317703008652, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.20735082030296326, + "drift/rejected_abs": 0.07366625219583511, + "epoch": 0.27175208581644816, + "grad_norm": 2.21875, + "learning_rate": 5.952430701883781e-10, + "logits/chosen": -2.3780345916748047, + "logits/rejected": -2.365642547607422, + "logps/chosen": -0.27689987421035767, + "logps/rejected": -0.2746134400367737, + "loss": 0.4126233160495758, + "loss/core": 0.40223345160484314, + "loss/preference_sft": 0.27305346727371216, + "loss/reference_anchor": 0.2695460915565491, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.073309898376465, + "rewards/margins": 1.336647391319275, + "rewards/rejected": 0.7366625070571899, + "ronpo/logit": 0.13366474211215973, + "ronpo/residual": 0.1408972442150116, + "ronpo/residual_abs": 0.18301846086978912, + "ronpo/target": -0.0072325123474001884, + "ronpo/target_abs": 0.07463272660970688, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.1881171017885208, + "drift/rejected_abs": 0.10389295965433121, + "epoch": 0.27334127930075486, + "grad_norm": 24.875, + "learning_rate": 4.731331010133288e-10, + "logits/chosen": -2.2028229236602783, + "logits/rejected": -2.12843656539917, + "logps/chosen": -0.2806995213031769, + "logps/rejected": -0.2797695994377136, + "loss": 0.1201997771859169, + "loss/core": 0.11440330743789673, + "loss/preference_sft": 0.28371673822402954, + "loss/reference_anchor": 0.13724195957183838, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8811709880828857, + "rewards/margins": 0.8511055111885071, + "rewards/rejected": 1.0300655364990234, + "ronpo/logit": 0.08511054515838623, + "ronpo/residual": 0.07745916396379471, + "ronpo/residual_abs": 0.1381129026412964, + "ronpo/target": 0.007651382125914097, + "ronpo/target_abs": 0.06799162924289703, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.24889037013053894, + "drift/rejected_abs": 0.12972140312194824, + "epoch": 0.2749304727850616, + "grad_norm": 0.75, + "learning_rate": 3.649474221911114e-10, + "logits/chosen": -2.090186595916748, + "logits/rejected": -2.0353407859802246, + "logps/chosen": -0.27941226959228516, + "logps/rejected": -0.28891220688819885, + "loss": 0.11919602006673813, + "loss/core": 0.11207473278045654, + "loss/preference_sft": 0.2893761694431305, + "loss/reference_anchor": 0.1745276153087616, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.488442897796631, + "rewards/margins": 1.1927058696746826, + "rewards/rejected": 1.2957369089126587, + "ronpo/logit": 0.11927058547735214, + "ronpo/residual": 0.11880163848400116, + "ronpo/residual_abs": 0.18198655545711517, + "ronpo/target": 0.0004689468187279999, + "ronpo/target_abs": 0.07780329138040543, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.46323925256729126, + "drift/rejected_abs": 0.21974530816078186, + "epoch": 0.2765196662693683, + "grad_norm": 532.0, + "learning_rate": 2.7072671793106074e-10, + "logits/chosen": -2.0998735427856445, + "logits/rejected": -2.087890625, + "logps/chosen": -0.26934343576431274, + "logps/rejected": -0.27645033597946167, + "loss": 0.7379623651504517, + "loss/core": 0.7018780708312988, + "loss/preference_sft": 0.2766755223274231, + "loss/reference_anchor": 1.0033100843429565, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.631134986877441, + "rewards/margins": 2.4353508949279785, + "rewards/rejected": 2.195784091949463, + "ronpo/logit": 0.24353507161140442, + "ronpo/residual": 0.2610352635383606, + "ronpo/residual_abs": 0.31035056710243225, + "ronpo/target": -0.017500165849924088, + "ronpo/target_abs": 0.0813521146774292, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.3059462904930115, + "drift/rejected_abs": 0.15434163808822632, + "epoch": 0.278108859753675, + "grad_norm": 5.0, + "learning_rate": 1.9050642078670547e-10, + "logits/chosen": -1.9450798034667969, + "logits/rejected": -1.9134384393692017, + "logps/chosen": -0.2710108757019043, + "logps/rejected": -0.2715742588043213, + "loss": 0.22777685523033142, + "loss/core": 0.21646137535572052, + "loss/preference_sft": 0.27517765760421753, + "loss/reference_anchor": 0.2957829236984253, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0580132007598877, + "rewards/margins": 1.5153281688690186, + "rewards/rejected": 1.5426852703094482, + "ronpo/logit": 0.15153281390666962, + "ronpo/residual": 0.1544264703989029, + "ronpo/residual_abs": 0.21350125968456268, + "ronpo/target": -0.002893655328080058, + "ronpo/target_abs": 0.08072160184383392, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.24425086379051208, + "drift/rejected_abs": 0.0907929316163063, + "epoch": 0.27969805323798175, + "grad_norm": 5.90625, + "learning_rate": 1.2431669833102288e-10, + "logits/chosen": -2.1156883239746094, + "logits/rejected": -2.1574745178222656, + "logps/chosen": -0.27831846475601196, + "logps/rejected": -0.29058295488357544, + "loss": 0.42100924253463745, + "loss/core": 0.41138726472854614, + "loss/preference_sft": 0.2927462160587311, + "loss/reference_anchor": 0.2456396371126175, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4425082206726074, + "rewards/margins": 1.5455248355865479, + "rewards/rejected": 0.8969835042953491, + "ronpo/logit": 0.15455248951911926, + "ronpo/residual": 0.1457471400499344, + "ronpo/residual_abs": 0.20269238948822021, + "ronpo/target": 0.008805358782410622, + "ronpo/target_abs": 0.08658818900585175, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.26528865098953247, + "drift/rejected_abs": 0.1037355437874794, + "epoch": 0.28128724672228844, + "grad_norm": 5.625, + "learning_rate": 7.218244181167594e-11, + "logits/chosen": -2.1619465351104736, + "logits/rejected": -2.223400592803955, + "logps/chosen": -0.29743996262550354, + "logps/rejected": -0.2958257794380188, + "loss": 0.3847818076610565, + "loss/core": 0.36538928747177124, + "loss/preference_sft": 0.2988830506801605, + "loss/reference_anchor": 0.5241828560829163, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.6526153087615967, + "rewards/margins": 1.6183156967163086, + "rewards/rejected": 1.034299612045288, + "ronpo/logit": 0.1618315875530243, + "ronpo/residual": 0.1641121506690979, + "ronpo/residual_abs": 0.20270681381225586, + "ronpo/target": -0.0022805710323154926, + "ronpo/target_abs": 0.07088293880224228, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.2995665669441223, + "drift/rejected_abs": 0.12155304849147797, + "epoch": 0.28287644020659514, + "grad_norm": 3.765625, + "learning_rate": 3.4123256790405397e-11, + "logits/chosen": -2.307978868484497, + "logits/rejected": -2.2648723125457764, + "logps/chosen": -0.2815799415111542, + "logps/rejected": -0.2809089720249176, + "loss": 0.4028624892234802, + "loss/core": 0.3849177956581116, + "loss/preference_sft": 0.286651074886322, + "loss/reference_anchor": 0.48404186964035034, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.99299693107605, + "rewards/margins": 1.7814449071884155, + "rewards/rejected": 1.2115522623062134, + "ronpo/logit": 0.17814448475837708, + "ronpo/residual": 0.18465931713581085, + "ronpo/residual_abs": 0.22240400314331055, + "ronpo/target": -0.006514822132885456, + "ronpo/target_abs": 0.07123463600873947, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.30803316831588745, + "drift/rejected_abs": 0.12433400005102158, + "epoch": 0.2844656336909019, + "grad_norm": 11.5, + "learning_rate": 1.0153455770185725e-11, + "logits/chosen": -2.0798468589782715, + "logits/rejected": -2.068073272705078, + "logps/chosen": -0.2840914726257324, + "logps/rejected": -0.2947894334793091, + "loss": 0.3525415360927582, + "loss/core": 0.33794763684272766, + "loss/preference_sft": 0.29777130484580994, + "loss/reference_anchor": 0.38719049096107483, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0781116485595703, + "rewards/margins": 1.9379209280014038, + "rewards/rejected": 1.1401903629302979, + "ronpo/logit": 0.19379207491874695, + "ronpo/residual": 0.1745019257068634, + "ronpo/residual_abs": 0.2425132542848587, + "ronpo/target": 0.019290190190076828, + "ronpo/target_abs": 0.08941071480512619, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.16016218066215515, + "drift/rejected_abs": 0.08440212905406952, + "epoch": 0.2860548271752086, + "grad_norm": 15.3125, + "learning_rate": 2.820528128971711e-13, + "logits/chosen": -2.104757308959961, + "logits/rejected": -2.0879600048065186, + "logps/chosen": -0.31782156229019165, + "logps/rejected": -0.3181937634944916, + "loss": 0.14113518595695496, + "loss/core": 0.13576209545135498, + "loss/preference_sft": 0.3204381763935089, + "loss/reference_anchor": 0.12147295475006104, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6012519598007202, + "rewards/margins": 0.888868510723114, + "rewards/rejected": 0.7123832106590271, + "ronpo/logit": 0.08888684213161469, + "ronpo/residual": 0.08423057198524475, + "ronpo/residual_abs": 0.14461413025856018, + "ronpo/target": 0.004656284116208553, + "ronpo/target_abs": 0.07319948077201843, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.4220078429910872, + "train_runtime": 7042.2326, + "train_samples_per_second": 2.045, + "train_steps_per_second": 0.128 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..89b7d88 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4952a6617b5a3d1bf987bab40bf4adb5bc18187abe394bc2181f9cba57402b68 +size 7057